[llvm] [CodeGen] computeRegisterProperties - fix `bf16` to `i16` bitcast quieting NaNs (PR #224867)
Folkert de Vries via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 12:04:51 PDT 2026
https://github.com/folkertdev updated https://github.com/llvm/llvm-project/pull/224867
>From 1a834122502f4b13320c9bc9738109caa8d3848a Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 20 Sep 2026 01:19:56 +0200
Subject: [PATCH 1/5] fix `bf16` to `i16` bitcast quieting NaNs
by generally bringing it more in line with f16
---
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +
llvm/lib/CodeGen/TargetLoweringBase.cpp | 12 +-
llvm/lib/Target/ARM/ARMISelLowering.h | 1 +
.../LoongArch/LoongArchISelLowering.cpp | 8 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 14 +
llvm/test/CodeGen/ARM/bf16-instructions.ll | 29 +
llvm/test/CodeGen/RISCV/bfloat.ll | 66 +
llvm/test/CodeGen/X86/bf16-fast-isel.ll | 38 +-
.../X86/bfloat-calling-conv-no-sse2.ll | 1434 ++++-------------
llvm/test/CodeGen/X86/bfloat-calling-conv.ll | 890 +++-------
.../test/CodeGen/X86/bfloat-int64-cvt-i686.ll | 26 -
llvm/test/CodeGen/X86/bfloat.ll | 422 ++---
12 files changed, 898 insertions(+), 2048 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c08d0e53ec34b..1551f833cbccf 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -571,6 +571,12 @@ class LLVM_ABI TargetLoweringBase {
// values are always passed and returned as f32.
virtual bool useFPRegsForHalfType() const { return false; }
+ // Return true if, for soft-promoted bfloat, the bfloat type should be
+ // passed to and returned from functions as f32. The default behavior is to
+ // pass as i16. If soft-promoted bfloat is not used, this function is
+ // ignored and values are always passed and returned as f32.
+ virtual bool useFPRegsForBFloat16Type() const { return false; }
+
// There are two general methods for expanding a BUILD_VECTOR node:
// 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
// them together.
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f852f7551ce16..31337cefbb735 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1537,8 +1537,16 @@ void TargetLoweringBase::computeRegisterProperties(
// promote it to f32, because there are no bf16 library calls (except for
// converting from f32 to bf16).
if (!isTypeLegal(MVT::bf16)) {
- NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
- RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
+ // Allow targets to control how we legalize bfloat.
+ bool UseFPRegsForBFloat16Type = useFPRegsForBFloat16Type();
+
+ if (!UseFPRegsForBFloat16Type) {
+ NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::i16];
+ RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::i16];
+ } else {
+ NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
+ RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
+ }
TransformToType[MVT::bf16] = MVT::f32;
ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
}
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 4f95a1cdeb2ef..30838f1772b4a 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -504,6 +504,7 @@ class VectorType;
Value *Accumulator = nullptr) const override;
bool useFPRegsForHalfType() const override { return true; }
+ bool useFPRegsForBFloat16Type() const override { return true; }
protected:
std::pair<const TargetRegisterClass *, uint8_t>
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 07f6a438bbb1b..bd871c52c7569 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -11837,6 +11837,10 @@ MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
if (VT == MVT::f16 && Subtarget.hasBasicF())
return MVT::f32;
+ // Use f32 to pass bf16.
+ if (VT == MVT::bf16 && Subtarget.hasBasicF())
+ return MVT::f32;
+
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
@@ -11846,6 +11850,10 @@ unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
if (VT == MVT::f16 && Subtarget.hasBasicF())
return 1;
+ // Use f32 to pass bf16.
+ if (VT == MVT::bf16 && Subtarget.hasBasicF())
+ return 1;
+
return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 2c05e3000c5eb..0ad546140554f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2961,6 +2961,13 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
!Subtarget.hasStdExtZfhminOrZhinxmin())
return MVT::f32;
+ // Use f32 to pass bf16 if it is legal and Zfbfmin/XAndesBFHCvt is not
+ // enabled. We might still end up using a GPR but that will be decided based
+ // on ABI.
+ if (VT == MVT::bf16 && Subtarget.hasStdExtFOrZfinx() &&
+ !Subtarget.hasStdExtZfbfmin() && !Subtarget.hasVendorXAndesBFHCvt())
+ return MVT::f32;
+
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
@@ -2984,6 +2991,13 @@ unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context
!Subtarget.hasStdExtZfhminOrZhinxmin())
return 1;
+ // Use f32 to pass bf16 if it is legal and Zfbfmin/XAndesBFHCvt is not
+ // enabled. We might still end up using a GPR but that will be decided based
+ // on ABI.
+ if (VT == MVT::bf16 && Subtarget.hasStdExtFOrZfinx() &&
+ !Subtarget.hasStdExtZfbfmin() && !Subtarget.hasVendorXAndesBFHCvt())
+ return 1;
+
return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
diff --git a/llvm/test/CodeGen/ARM/bf16-instructions.ll b/llvm/test/CodeGen/ARM/bf16-instructions.ll
index c4d51fd0224aa..e6527050d5228 100644
--- a/llvm/test/CodeGen/ARM/bf16-instructions.ll
+++ b/llvm/test/CodeGen/ARM/bf16-instructions.ll
@@ -419,6 +419,35 @@ define i16 @test_bitcast_bfloattoi16(bfloat %a) #0 {
ret i16 %r
}
+define void @test_bitcast_bfloattoi16_in_branch(bfloat %a, i1 %c, ptr %p) #0 {
+; CHECK-CVT-LABEL: test_bitcast_bfloattoi16_in_branch:
+; CHECK-CVT: @ %bb.0: @ %entry
+; CHECK-CVT-NEXT: tst r0, #1
+; CHECK-CVT-NEXT: vmovne r0, s0
+; CHECK-CVT-NEXT: strhne r0, [r1]
+; CHECK-CVT-NEXT: bx lr
+;
+; CHECK-BF16-LABEL: test_bitcast_bfloattoi16_in_branch:
+; CHECK-BF16: @ %bb.0: @ %entry
+; CHECK-BF16-NEXT: .pad #4
+; CHECK-BF16-NEXT: sub sp, sp, #4
+; CHECK-BF16-NEXT: vmov r2, s0
+; CHECK-BF16-NEXT: tst r0, #1
+; CHECK-BF16-NEXT: strh r2, [sp, #2]
+; CHECK-BF16-NEXT: ldrhne r0, [sp, #2]
+; CHECK-BF16-NEXT: strhne r0, [r1]
+; CHECK-BF16-NEXT: add sp, sp, #4
+; CHECK-BF16-NEXT: bx lr
+entry:
+ br i1 %c, label %bitcast, label %exit
+bitcast:
+ %b = bitcast bfloat %a to i16
+ store i16 %b, ptr %p
+ br label %exit
+exit:
+ ret void
+}
+
define bfloat @test_bitcast_i16tobfloat(i16 %a) #0 {
; CHECK-CVT-LABEL: test_bitcast_i16tobfloat:
; CHECK-CVT: @ %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/bfloat.ll b/llvm/test/CodeGen/RISCV/bfloat.ll
index e43c7f55ac612..50eb981898643 100644
--- a/llvm/test/CodeGen/RISCV/bfloat.ll
+++ b/llvm/test/CodeGen/RISCV/bfloat.ll
@@ -312,6 +312,72 @@ define i16 @bfloat_to_i16(bfloat %a) nounwind {
ret i16 %1
}
+define void @bfloat_to_i16_in_branch(bfloat %a, i1 %c, ptr %p) nounwind {
+; RV32I-ILP32-LABEL: bfloat_to_i16_in_branch:
+; RV32I-ILP32: # %bb.0: # %entry
+; RV32I-ILP32-NEXT: andi a1, a1, 1
+; RV32I-ILP32-NEXT: beqz a1, .LBB6_2
+; RV32I-ILP32-NEXT: # %bb.1: # %bitcast
+; RV32I-ILP32-NEXT: sh a0, 0(a2)
+; RV32I-ILP32-NEXT: .LBB6_2: # %exit
+; RV32I-ILP32-NEXT: ret
+;
+; RV64I-LP64-LABEL: bfloat_to_i16_in_branch:
+; RV64I-LP64: # %bb.0: # %entry
+; RV64I-LP64-NEXT: andi a1, a1, 1
+; RV64I-LP64-NEXT: beqz a1, .LBB6_2
+; RV64I-LP64-NEXT: # %bb.1: # %bitcast
+; RV64I-LP64-NEXT: sh a0, 0(a2)
+; RV64I-LP64-NEXT: .LBB6_2: # %exit
+; RV64I-LP64-NEXT: ret
+;
+; RV32ID-ILP32-LABEL: bfloat_to_i16_in_branch:
+; RV32ID-ILP32: # %bb.0: # %entry
+; RV32ID-ILP32-NEXT: andi a1, a1, 1
+; RV32ID-ILP32-NEXT: beqz a1, .LBB6_2
+; RV32ID-ILP32-NEXT: # %bb.1: # %bitcast
+; RV32ID-ILP32-NEXT: sh a0, 0(a2)
+; RV32ID-ILP32-NEXT: .LBB6_2: # %exit
+; RV32ID-ILP32-NEXT: ret
+;
+; RV64ID-LP64-LABEL: bfloat_to_i16_in_branch:
+; RV64ID-LP64: # %bb.0: # %entry
+; RV64ID-LP64-NEXT: andi a1, a1, 1
+; RV64ID-LP64-NEXT: beqz a1, .LBB6_2
+; RV64ID-LP64-NEXT: # %bb.1: # %bitcast
+; RV64ID-LP64-NEXT: sh a0, 0(a2)
+; RV64ID-LP64-NEXT: .LBB6_2: # %exit
+; RV64ID-LP64-NEXT: ret
+;
+; RV32ID-ILP32D-LABEL: bfloat_to_i16_in_branch:
+; RV32ID-ILP32D: # %bb.0: # %entry
+; RV32ID-ILP32D-NEXT: andi a0, a0, 1
+; RV32ID-ILP32D-NEXT: beqz a0, .LBB6_2
+; RV32ID-ILP32D-NEXT: # %bb.1: # %bitcast
+; RV32ID-ILP32D-NEXT: fmv.x.w a0, fa0
+; RV32ID-ILP32D-NEXT: sh a0, 0(a1)
+; RV32ID-ILP32D-NEXT: .LBB6_2: # %exit
+; RV32ID-ILP32D-NEXT: ret
+;
+; RV64ID-LP64D-LABEL: bfloat_to_i16_in_branch:
+; RV64ID-LP64D: # %bb.0: # %entry
+; RV64ID-LP64D-NEXT: andi a0, a0, 1
+; RV64ID-LP64D-NEXT: beqz a0, .LBB6_2
+; RV64ID-LP64D-NEXT: # %bb.1: # %bitcast
+; RV64ID-LP64D-NEXT: fmv.x.w a0, fa0
+; RV64ID-LP64D-NEXT: sh a0, 0(a1)
+; RV64ID-LP64D-NEXT: .LBB6_2: # %exit
+; RV64ID-LP64D-NEXT: ret
+entry:
+ br i1 %c, label %bitcast, label %exit
+bitcast:
+ %b = bitcast bfloat %a to i16
+ store i16 %b, ptr %p
+ br label %exit
+exit:
+ ret void
+}
+
define bfloat @bfloat_add(bfloat %a, bfloat %b) nounwind {
; RV32I-ILP32-LABEL: bfloat_add:
; RV32I-ILP32: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/bf16-fast-isel.ll b/llvm/test/CodeGen/X86/bf16-fast-isel.ll
index 1f36d258a1a58..7fb52749615b9 100644
--- a/llvm/test/CodeGen/X86/bf16-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/bf16-fast-isel.ll
@@ -6,8 +6,8 @@ define i8 @test_direct_call(ptr %f) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq foo at PLT
-; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq bar at PLT
; CHECK-NEXT: popq %rcx
; CHECK-NEXT: retq
@@ -22,8 +22,8 @@ define i8 @test_fast_direct_call(ptr %f) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq foo_fast at PLT
-; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq bar at PLT
; CHECK-NEXT: popq %rcx
; CHECK-NEXT: retq
@@ -40,8 +40,8 @@ define i8 @test_indirect_all(ptr %fptr, ptr %f) nounwind {
; CHECK-NEXT: movq %rdi, %rbx
; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: callq foo at PLT
-; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
@@ -64,8 +64,8 @@ define i8 @test_indirect_all2(ptr %fptr, ptr %f, i1 %cond) nounwind {
; CHECK-NEXT: testb $1, %bpl
; CHECK-NEXT: je .LBB3_2
; CHECK-NEXT: # %bb.1: # %exit
-; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: jmp .LBB3_3
; CHECK-NEXT: .LBB3_2: # %exit2
@@ -95,8 +95,8 @@ define i8 @test_fast_indirect_all(ptr %fptr, ptr %f) nounwind {
; CHECK-NEXT: movq %rdi, %rbx
; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: callq foo at PLT
-; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
+; CHECK-NEXT: pextrw $0, %xmm0, %eax
+; CHECK-NEXT: pinsrw $0, %eax, %xmm0
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
@@ -112,20 +112,14 @@ declare void @take_bfloats({ bfloat, bfloat })
define void @call_get_bfloats() nounwind {
; CHECK-LABEL: call_get_bfloats:
; CHECK: # %bb.0:
-; CHECK-NEXT: subq $40, %rsp
+; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq get_bfloats at PLT
-; CHECK-NEXT: pslld $16, %xmm1
-; CHECK-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT: pslld $16, %xmm0
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT: callq __truncsfbf2 at PLT
-; CHECK-NEXT: movaps %xmm0, %xmm1
-; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: pextrw $0, %xmm1, %eax
+; CHECK-NEXT: pextrw $0, %xmm0, %ecx
+; CHECK-NEXT: pinsrw $0, %ecx, %xmm0
+; CHECK-NEXT: pinsrw $0, %eax, %xmm1
; CHECK-NEXT: callq take_bfloats at PLT
-; CHECK-NEXT: addq $40, %rsp
+; CHECK-NEXT: popq %rax
; CHECK-NEXT: retq
%res = call { bfloat, bfloat } @get_bfloats()
call void @take_bfloats({ bfloat, bfloat } %res)
diff --git a/llvm/test/CodeGen/X86/bfloat-calling-conv-no-sse2.ll b/llvm/test/CodeGen/X86/bfloat-calling-conv-no-sse2.ll
index f363cad816dfb..a06623d6357ba 100644
--- a/llvm/test/CodeGen/X86/bfloat-calling-conv-no-sse2.ll
+++ b/llvm/test/CodeGen/X86/bfloat-calling-conv-no-sse2.ll
@@ -7,7 +7,7 @@
define bfloat @return_arg_bf16(bfloat %x) #0 {
; CHECK-LABEL: return_arg_bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; CHECK-NEXT: retl
ret bfloat %x
}
@@ -15,479 +15,103 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
define <2 x bfloat> @return_arg_v2bf16(<2 x bfloat> %x) #0 {
; CHECK-LABEL: return_arg_v2bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: flds {{[0-9]+}}(%esp)
-; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; CHECK-NEXT: retl
ret <2 x bfloat> %x
}
define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
-; NOSSE-LABEL: return_arg_v3bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: pushl %eax
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %esi
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: # kill: def $ax killed $ax def $eax
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movzwl %si, %edi
-; NOSSE-NEXT: orl %eax, %edi
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, 4(%esi)
-; NOSSE-NEXT: movl %edi, (%esi)
-; NOSSE-NEXT: movl %esi, %eax
-; NOSSE-NEXT: addl $4, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: retl $4
-;
-; SSE-LABEL: return_arg_v3bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: pushl %eax
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %esi
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: # kill: def $ax killed $ax def $eax
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movzwl %si, %edi
-; SSE-NEXT: orl %eax, %edi
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, 4(%esi)
-; SSE-NEXT: movl %edi, (%esi)
-; SSE-NEXT: movl %esi, %eax
-; SSE-NEXT: addl $4, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: retl $4
+; CHECK-LABEL: return_arg_v3bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: retl
ret <3 x bfloat> %x
}
define <4 x bfloat> @return_arg_v4bf16(<4 x bfloat> %x) #0 {
-; NOSSE-LABEL: return_arg_v4bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %ebp
-; NOSSE-NEXT: pushl %ebx
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $12, %esp
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %esi
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %edi
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %ebx
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, 6(%ebp)
-; NOSSE-NEXT: movw %bx, 4(%ebp)
-; NOSSE-NEXT: movw %di, 2(%ebp)
-; NOSSE-NEXT: movw %si, (%ebp)
-; NOSSE-NEXT: movl %ebp, %eax
-; NOSSE-NEXT: addl $12, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: popl %ebx
-; NOSSE-NEXT: popl %ebp
-; NOSSE-NEXT: retl $4
-;
-; SSE-LABEL: return_arg_v4bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %ebp
-; SSE-NEXT: pushl %ebx
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $12, %esp
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %esi
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %edi
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %ebx
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, 6(%ebp)
-; SSE-NEXT: movw %bx, 4(%ebp)
-; SSE-NEXT: movw %di, 2(%ebp)
-; SSE-NEXT: movw %si, (%ebp)
-; SSE-NEXT: movl %ebp, %eax
-; SSE-NEXT: addl $12, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: popl %ebx
-; SSE-NEXT: popl %ebp
-; SSE-NEXT: retl $4
+; CHECK-LABEL: return_arg_v4bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT: movw %di, 6(%eax)
+; CHECK-NEXT: movw %si, 4(%eax)
+; CHECK-NEXT: movw %dx, 2(%eax)
+; CHECK-NEXT: movw %cx, (%eax)
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: popl %edi
+; CHECK-NEXT: retl $4
ret <4 x bfloat> %x
}
define <8 x bfloat> @return_arg_v8bf16(<8 x bfloat> %x) #0 {
-; NOSSE-LABEL: return_arg_v8bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %ebp
-; NOSSE-NEXT: pushl %ebx
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $12, %esp
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %esi
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %edi
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %ebx
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, 14(%ebp)
-; NOSSE-NEXT: movw %bx, 12(%ebp)
-; NOSSE-NEXT: movw %di, 10(%ebp)
-; NOSSE-NEXT: movw %si, 8(%ebp)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 6(%ebp)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 4(%ebp)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 2(%ebp)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, (%ebp)
-; NOSSE-NEXT: movl %ebp, %eax
-; NOSSE-NEXT: addl $12, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: popl %ebx
-; NOSSE-NEXT: popl %ebp
-; NOSSE-NEXT: retl $4
-;
-; SSE-LABEL: return_arg_v8bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %ebp
-; SSE-NEXT: pushl %ebx
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $12, %esp
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %esi
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %edi
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %ebx
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, 14(%ebp)
-; SSE-NEXT: movw %bx, 12(%ebp)
-; SSE-NEXT: movw %di, 10(%ebp)
-; SSE-NEXT: movw %si, 8(%ebp)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 6(%ebp)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 4(%ebp)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 2(%ebp)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, (%ebp)
-; SSE-NEXT: movl %ebp, %eax
-; SSE-NEXT: addl $12, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: popl %ebx
-; SSE-NEXT: popl %ebp
-; SSE-NEXT: retl $4
+; CHECK-LABEL: return_arg_v8bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 14(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 12(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 10(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 8(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 6(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 4(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 2(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, (%eax)
+; CHECK-NEXT: retl $4
ret <8 x bfloat> %x
}
define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
-; NOSSE-LABEL: return_arg_v16bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %ebp
-; NOSSE-NEXT: pushl %ebx
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $28, %esp
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %esi
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %ebx
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %ebp
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, 30(%edi)
-; NOSSE-NEXT: movw %bp, 28(%edi)
-; NOSSE-NEXT: movw %bx, 26(%edi)
-; NOSSE-NEXT: movw %si, 24(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 22(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 20(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 18(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 16(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 14(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 12(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 10(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 8(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 6(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 4(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, 2(%edi)
-; NOSSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; NOSSE-NEXT: movw %ax, (%edi)
-; NOSSE-NEXT: movl %edi, %eax
-; NOSSE-NEXT: addl $28, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: popl %ebx
-; NOSSE-NEXT: popl %ebp
-; NOSSE-NEXT: retl $4
-;
-; SSE-LABEL: return_arg_v16bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %ebp
-; SSE-NEXT: pushl %ebx
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $28, %esp
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %esi
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %ebx
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %ebp
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %edi
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, 30(%edi)
-; SSE-NEXT: movw %bp, 28(%edi)
-; SSE-NEXT: movw %bx, 26(%edi)
-; SSE-NEXT: movw %si, 24(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 22(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 20(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 18(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 16(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 14(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 12(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 10(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 8(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 6(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 4(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, 2(%edi)
-; SSE-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
-; SSE-NEXT: movw %ax, (%edi)
-; SSE-NEXT: movl %edi, %eax
-; SSE-NEXT: addl $28, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: popl %ebx
-; SSE-NEXT: popl %ebp
-; SSE-NEXT: retl $4
+; CHECK-LABEL: return_arg_v16bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 30(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 28(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 26(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 24(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 22(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 20(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 18(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 16(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 14(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 12(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 10(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 8(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 6(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 4(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, 2(%eax)
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movw %cx, (%eax)
+; CHECK-NEXT: retl $4
ret <16 x bfloat> %x
}
@@ -499,41 +123,18 @@ declare <8 x bfloat> @returns_v8bf16(<8 x bfloat>)
declare <16 x bfloat> @returns_v16bf16(<16 x bfloat>)
define void @call_ret_bf16(ptr %ptr) #0 {
-; NOSSE-LABEL: call_ret_bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $8, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; NOSSE-NEXT: movzwl (%esi), %eax
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll returns_bf16 at PLT
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, (%esi)
-; NOSSE-NEXT: addl $8, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: retl
-;
-; SSE-LABEL: call_ret_bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $8, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; SSE-NEXT: movzwl (%esi), %eax
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll returns_bf16 at PLT
-; SSE-NEXT: fstps (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, (%esi)
-; SSE-NEXT: addl $8, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: retl
+; CHECK-LABEL: call_ret_bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $8, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movzwl (%esi), %eax
+; CHECK-NEXT: movl %eax, (%esp)
+; CHECK-NEXT: calll returns_bf16 at PLT
+; CHECK-NEXT: movw %ax, (%esi)
+; CHECK-NEXT: addl $8, %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: retl
%val = load bfloat, ptr %ptr
%bf16 = call bfloat @returns_bf16(bfloat %val)
store bfloat %bf16, ptr %ptr
@@ -541,69 +142,22 @@ define void @call_ret_bf16(ptr %ptr) #0 {
}
define void @call_ret_v2bf16(ptr %ptr) #0 {
-; NOSSE-LABEL: call_ret_v2bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $20, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi
-; NOSSE-NEXT: movzwl 2(%edi), %eax
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl (%edi), %eax
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll returns_v2bf16 at PLT
-; NOSSE-NEXT: fxch %st(1)
-; NOSSE-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movl %eax, %esi
-; NOSSE-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; NOSSE-NEXT: fstps (%esp)
-; NOSSE-NEXT: calll __truncsfbf2
-; NOSSE-NEXT: movw %ax, 2(%edi)
-; NOSSE-NEXT: movw %si, (%edi)
-; NOSSE-NEXT: addl $20, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: retl
-;
-; SSE-LABEL: call_ret_v2bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $36, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %edi
-; SSE-NEXT: movzwl 2(%edi), %eax
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl (%edi), %eax
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, (%esp)
-; SSE-NEXT: calll returns_v2bf16 at PLT
-; SSE-NEXT: fxch %st(1)
-; SSE-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
-; SSE-NEXT: fstps (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movl %eax, %esi
-; SSE-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
-; SSE-NEXT: fstps (%esp)
-; SSE-NEXT: calll __truncsfbf2
-; SSE-NEXT: movw %ax, 2(%edi)
-; SSE-NEXT: movw %si, (%edi)
-; SSE-NEXT: addl $36, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: retl
+; CHECK-LABEL: call_ret_v2bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $8, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movzwl 2(%esi), %eax
+; CHECK-NEXT: subl $8, %esp
+; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: pushl (%esi)
+; CHECK-NEXT: calll returns_v2bf16 at PLT
+; CHECK-NEXT: addl $16, %esp
+; CHECK-NEXT: movw %dx, 2(%esi)
+; CHECK-NEXT: movw %ax, (%esi)
+; CHECK-NEXT: addl $8, %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: retl
%val = load <2 x bfloat>, ptr %ptr
%bf16 = call <2 x bfloat> @returns_v2bf16(<2 x bfloat> %val)
store <2 x bfloat> %bf16, ptr %ptr
@@ -611,69 +165,29 @@ define void @call_ret_v2bf16(ptr %ptr) #0 {
}
define void @call_ret_v3bf16(ptr %ptr) #0 {
-; NOSSE-LABEL: call_ret_v3bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $40, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; NOSSE-NEXT: movl (%esi), %eax
-; NOSSE-NEXT: movl 4(%esi), %ecx
-; NOSSE-NEXT: leal {{[0-9]+}}(%esp), %edx
-; NOSSE-NEXT: movl %edx, (%esp)
-; NOSSE-NEXT: shll $16, %ecx
-; NOSSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl %eax, %ecx
-; NOSSE-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
-; NOSSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: calll returns_v3bf16 at PLT
-; NOSSE-NEXT: subl $4, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; NOSSE-NEXT: movw %cx, 4(%esi)
-; NOSSE-NEXT: movl %eax, (%esi)
-; NOSSE-NEXT: addl $40, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: retl
-;
-; SSE-LABEL: call_ret_v3bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $40, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; SSE-NEXT: movl (%esi), %eax
-; SSE-NEXT: movl 4(%esi), %ecx
-; SSE-NEXT: leal {{[0-9]+}}(%esp), %edx
-; SSE-NEXT: movl %edx, (%esp)
-; SSE-NEXT: shll $16, %ecx
-; SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl %eax, %ecx
-; SSE-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
-; SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: calll returns_v3bf16 at PLT
-; SSE-NEXT: subl $4, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; SSE-NEXT: movw %cx, 4(%esi)
-; SSE-NEXT: movl %eax, (%esi)
-; SSE-NEXT: addl $40, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: retl
+; CHECK-LABEL: call_ret_v3bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $8, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movl (%esi), %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: subl $4, %esp
+; CHECK-NEXT: pushl 4(%esi)
+; CHECK-NEXT: pushl %ecx
+; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: calll returns_v3bf16 at PLT
+; CHECK-NEXT: addl $16, %esp
+; CHECK-NEXT: # kill: def $dx killed $dx def $edx
+; CHECK-NEXT: shll $16, %edx
+; CHECK-NEXT: movzwl %ax, %eax
+; CHECK-NEXT: orl %edx, %eax
+; CHECK-NEXT: movw %cx, 4(%esi)
+; CHECK-NEXT: movl %eax, (%esi)
+; CHECK-NEXT: addl $8, %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: retl
%val = load <3 x bfloat>, ptr %ptr
%bf16 = call <3 x bfloat> @returns_v3bf16(<3 x bfloat> %val)
store <3 x bfloat> %bf16, ptr %ptr
@@ -681,91 +195,31 @@ define void @call_ret_v3bf16(ptr %ptr) #0 {
}
define void @call_ret_v4bf16(ptr %ptr) #0 {
-; NOSSE-LABEL: call_ret_v4bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %ebx
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $48, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; NOSSE-NEXT: movzwl 2(%esi), %ecx
-; NOSSE-NEXT: movl (%esi), %eax
-; NOSSE-NEXT: movl 4(%esi), %edx
-; NOSSE-NEXT: movzwl 6(%esi), %edi
-; NOSSE-NEXT: leal {{[0-9]+}}(%esp), %ebx
-; NOSSE-NEXT: movl %ebx, (%esp)
-; NOSSE-NEXT: shll $16, %edi
-; NOSSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %edx
-; NOSSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %ecx
-; NOSSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: calll returns_v4bf16 at PLT
-; NOSSE-NEXT: subl $4, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; NOSSE-NEXT: movw %dx, 6(%esi)
-; NOSSE-NEXT: movw %cx, 4(%esi)
-; NOSSE-NEXT: movl %eax, (%esi)
-; NOSSE-NEXT: addl $48, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: popl %ebx
-; NOSSE-NEXT: retl
-;
-; SSE-LABEL: call_ret_v4bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %ebx
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $48, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; SSE-NEXT: movzwl 2(%esi), %ecx
-; SSE-NEXT: movl (%esi), %eax
-; SSE-NEXT: movl 4(%esi), %edx
-; SSE-NEXT: movzwl 6(%esi), %edi
-; SSE-NEXT: leal {{[0-9]+}}(%esp), %ebx
-; SSE-NEXT: movl %ebx, (%esp)
-; SSE-NEXT: shll $16, %edi
-; SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %edx
-; SSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %ecx
-; SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: calll returns_v4bf16 at PLT
-; SSE-NEXT: subl $4, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; SSE-NEXT: movw %dx, 6(%esi)
-; SSE-NEXT: movw %cx, 4(%esi)
-; SSE-NEXT: movl %eax, (%esi)
-; SSE-NEXT: addl $48, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: popl %ebx
-; SSE-NEXT: retl
+; CHECK-LABEL: call_ret_v4bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $8, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movzwl 2(%esi), %eax
+; CHECK-NEXT: movzwl 6(%esi), %ecx
+; CHECK-NEXT: subl $12, %esp
+; CHECK-NEXT: leal {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: pushl %ecx
+; CHECK-NEXT: pushl 4(%esi)
+; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: pushl (%esi)
+; CHECK-NEXT: pushl %edx
+; CHECK-NEXT: calll returns_v4bf16 at PLT
+; CHECK-NEXT: addl $28, %esp
+; CHECK-NEXT: movl (%esp), %eax
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movw %dx, 6(%esi)
+; CHECK-NEXT: movw %cx, 4(%esi)
+; CHECK-NEXT: movl %eax, (%esi)
+; CHECK-NEXT: addl $8, %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: retl
%val = load <4 x bfloat>, ptr %ptr
%bf16 = call <4 x bfloat> @returns_v4bf16(<4 x bfloat> %val)
store <4 x bfloat> %bf16, ptr %ptr
@@ -773,167 +227,53 @@ define void @call_ret_v4bf16(ptr %ptr) #0 {
}
define void @call_ret_v8bf16(ptr %ptr) #0 {
-; NOSSE-LABEL: call_ret_v8bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %ebp
-; NOSSE-NEXT: pushl %ebx
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: subl $108, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; NOSSE-NEXT: movzwl 2(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl (%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 4(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl 6(%esi), %edi
-; NOSSE-NEXT: movl 8(%esi), %ebx
-; NOSSE-NEXT: movzwl 10(%esi), %ebp
-; NOSSE-NEXT: movl 12(%esi), %ecx
-; NOSSE-NEXT: movzwl 14(%esi), %eax
-; NOSSE-NEXT: leal {{[0-9]+}}(%esp), %edx
-; NOSSE-NEXT: movl %edx, (%esp)
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %ecx
-; NOSSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %ebp
-; NOSSE-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %ebx
-; NOSSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %edi
-; NOSSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: calll returns_v8bf16 at PLT
-; NOSSE-NEXT: subl $4, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %edi
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
-; NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movw %ax, 14(%esi)
-; NOSSE-NEXT: movw %bp, 12(%esi)
-; NOSSE-NEXT: movw %bx, 10(%esi)
-; NOSSE-NEXT: movw %di, 8(%esi)
-; NOSSE-NEXT: movw %dx, 6(%esi)
-; NOSSE-NEXT: movw %cx, 4(%esi)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: movl %eax, (%esi)
-; NOSSE-NEXT: addl $108, %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: popl %ebx
-; NOSSE-NEXT: popl %ebp
-; NOSSE-NEXT: retl
-;
-; SSE-LABEL: call_ret_v8bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %ebp
-; SSE-NEXT: pushl %ebx
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: subl $108, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; SSE-NEXT: movzwl 2(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl (%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 4(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl 6(%esi), %edi
-; SSE-NEXT: movl 8(%esi), %ebx
-; SSE-NEXT: movzwl 10(%esi), %ebp
-; SSE-NEXT: movl 12(%esi), %ecx
-; SSE-NEXT: movzwl 14(%esi), %eax
-; SSE-NEXT: leal {{[0-9]+}}(%esp), %edx
-; SSE-NEXT: movl %edx, (%esp)
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %ecx
-; SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %ebp
-; SSE-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %ebx
-; SSE-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %edi
-; SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: calll returns_v8bf16 at PLT
-; SSE-NEXT: subl $4, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %edi
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
-; SSE-NEXT: movzwl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movw %ax, 14(%esi)
-; SSE-NEXT: movw %bp, 12(%esi)
-; SSE-NEXT: movw %bx, 10(%esi)
-; SSE-NEXT: movw %di, 8(%esi)
-; SSE-NEXT: movw %dx, 6(%esi)
-; SSE-NEXT: movw %cx, 4(%esi)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: movl %eax, (%esi)
-; SSE-NEXT: addl $108, %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: popl %ebx
-; SSE-NEXT: popl %ebp
-; SSE-NEXT: retl
+; CHECK-LABEL: call_ret_v8bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %ebp
+; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $28, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movzwl 2(%esi), %eax
+; CHECK-NEXT: movzwl 6(%esi), %edx
+; CHECK-NEXT: movzwl 10(%esi), %edi
+; CHECK-NEXT: movzwl 14(%esi), %ebx
+; CHECK-NEXT: subl $12, %esp
+; CHECK-NEXT: leal {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl 12(%esi)
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl 8(%esi)
+; CHECK-NEXT: pushl %edx
+; CHECK-NEXT: pushl 4(%esi)
+; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: pushl (%esi)
+; CHECK-NEXT: pushl %ecx
+; CHECK-NEXT: calll returns_v8bf16 at PLT
+; CHECK-NEXT: addl $44, %esp
+; CHECK-NEXT: movl (%esp), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movw %ax, 14(%esi)
+; CHECK-NEXT: movw %bp, 12(%esi)
+; CHECK-NEXT: movw %bx, 10(%esi)
+; CHECK-NEXT: movw %di, 8(%esi)
+; CHECK-NEXT: movw %dx, 6(%esi)
+; CHECK-NEXT: movw %cx, 4(%esi)
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT: movl %eax, (%esi)
+; CHECK-NEXT: addl $28, %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: popl %edi
+; CHECK-NEXT: popl %ebx
+; CHECK-NEXT: popl %ebp
+; CHECK-NEXT: retl
%val = load <8 x bfloat>, ptr %ptr
%bf16 = call <8 x bfloat> @returns_v8bf16(<8 x bfloat> %val)
store <8 x bfloat> %bf16, ptr %ptr
@@ -941,299 +281,76 @@ define void @call_ret_v8bf16(ptr %ptr) #0 {
}
define void @call_ret_v16bf16(ptr %ptr) #0 {
-; NOSSE-LABEL: call_ret_v16bf16:
-; NOSSE: # %bb.0:
-; NOSSE-NEXT: pushl %ebp
-; NOSSE-NEXT: movl %esp, %ebp
-; NOSSE-NEXT: pushl %ebx
-; NOSSE-NEXT: pushl %edi
-; NOSSE-NEXT: pushl %esi
-; NOSSE-NEXT: andl $-32, %esp
-; NOSSE-NEXT: subl $256, %esp # imm = 0x100
-; NOSSE-NEXT: movl 8(%ebp), %esi
-; NOSSE-NEXT: movzwl 2(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl (%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 4(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl 6(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 8(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl 10(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 12(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl 14(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 16(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl 18(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 20(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movzwl 22(%esi), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl 24(%esi), %edi
-; NOSSE-NEXT: movzwl 26(%esi), %edx
-; NOSSE-NEXT: movl 28(%esi), %ecx
-; NOSSE-NEXT: movzwl 30(%esi), %eax
-; NOSSE-NEXT: leal {{[0-9]+}}(%esp), %ebx
-; NOSSE-NEXT: movl %ebx, (%esp)
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %ecx
-; NOSSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %edx
-; NOSSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: shll $16, %edi
-; NOSSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: shll $16, %eax
-; NOSSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; NOSSE-NEXT: fstps {{[0-9]+}}(%esp)
-; NOSSE-NEXT: calll returns_v16bf16 at PLT
-; NOSSE-NEXT: subl $4, %esp
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edx
-; NOSSE-NEXT: movl %edx, 28(%esi)
-; NOSSE-NEXT: movl %eax, 24(%esi)
-; NOSSE-NEXT: movl %ecx, 20(%esi)
-; NOSSE-NEXT: movl %ebx, 16(%esi)
-; NOSSE-NEXT: movl %edi, 12(%esi)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: movl %eax, 8(%esi)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: movl %eax, 4(%esi)
-; NOSSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; NOSSE-NEXT: movl %eax, (%esi)
-; NOSSE-NEXT: leal -12(%ebp), %esp
-; NOSSE-NEXT: popl %esi
-; NOSSE-NEXT: popl %edi
-; NOSSE-NEXT: popl %ebx
-; NOSSE-NEXT: popl %ebp
-; NOSSE-NEXT: retl
-;
-; SSE-LABEL: call_ret_v16bf16:
-; SSE: # %bb.0:
-; SSE-NEXT: pushl %ebp
-; SSE-NEXT: movl %esp, %ebp
-; SSE-NEXT: pushl %ebx
-; SSE-NEXT: pushl %edi
-; SSE-NEXT: pushl %esi
-; SSE-NEXT: andl $-32, %esp
-; SSE-NEXT: subl $256, %esp # imm = 0x100
-; SSE-NEXT: movl 8(%ebp), %esi
-; SSE-NEXT: movzwl 2(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl (%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 4(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl 6(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 8(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl 10(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 12(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl 14(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 16(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl 18(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 20(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movzwl 22(%esi), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl 24(%esi), %edi
-; SSE-NEXT: movzwl 26(%esi), %edx
-; SSE-NEXT: movl 28(%esi), %ecx
-; SSE-NEXT: movzwl 30(%esi), %eax
-; SSE-NEXT: leal {{[0-9]+}}(%esp), %ebx
-; SSE-NEXT: movl %ebx, (%esp)
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %ecx
-; SSE-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %edx
-; SSE-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: shll $16, %edi
-; SSE-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: shll $16, %eax
-; SSE-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-NEXT: calll returns_v16bf16 at PLT
-; SSE-NEXT: subl $4, %esp
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %edi
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT: movl {{[0-9]+}}(%esp), %edx
-; SSE-NEXT: movl %edx, 28(%esi)
-; SSE-NEXT: movl %eax, 24(%esi)
-; SSE-NEXT: movl %ecx, 20(%esi)
-; SSE-NEXT: movl %ebx, 16(%esi)
-; SSE-NEXT: movl %edi, 12(%esi)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: movl %eax, 8(%esi)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: movl %eax, 4(%esi)
-; SSE-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; SSE-NEXT: movl %eax, (%esi)
-; SSE-NEXT: leal -12(%ebp), %esp
-; SSE-NEXT: popl %esi
-; SSE-NEXT: popl %edi
-; SSE-NEXT: popl %ebx
-; SSE-NEXT: popl %ebp
-; SSE-NEXT: retl
+; CHECK-LABEL: call_ret_v16bf16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushl %ebp
+; CHECK-NEXT: movl %esp, %ebp
+; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: andl $-32, %esp
+; CHECK-NEXT: subl $96, %esp
+; CHECK-NEXT: movl 8(%ebp), %esi
+; CHECK-NEXT: movzwl 2(%esi), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzwl 6(%esi), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzwl 10(%esi), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzwl 14(%esi), %ebx
+; CHECK-NEXT: movzwl 18(%esi), %edi
+; CHECK-NEXT: movzwl 22(%esi), %edx
+; CHECK-NEXT: movzwl 26(%esi), %ecx
+; CHECK-NEXT: movzwl 30(%esi), %eax
+; CHECK-NEXT: subl $12, %esp
+; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: pushl 28(%esi)
+; CHECK-NEXT: pushl %ecx
+; CHECK-NEXT: pushl 24(%esi)
+; CHECK-NEXT: pushl %edx
+; CHECK-NEXT: pushl 20(%esi)
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl 16(%esi)
+; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl 12(%esi)
+; CHECK-NEXT: pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: pushl 8(%esi)
+; CHECK-NEXT: pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: pushl 4(%esi)
+; CHECK-NEXT: pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: pushl (%esi)
+; CHECK-NEXT: leal {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: calll returns_v16bf16 at PLT
+; CHECK-NEXT: addl $76, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movl %edx, 28(%esi)
+; CHECK-NEXT: movl %eax, 24(%esi)
+; CHECK-NEXT: movl %ecx, 20(%esi)
+; CHECK-NEXT: movl %ebx, 16(%esi)
+; CHECK-NEXT: movl %edi, 12(%esi)
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT: movl %eax, 8(%esi)
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT: movl %eax, 4(%esi)
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT: movl %eax, (%esi)
+; CHECK-NEXT: leal -12(%ebp), %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: popl %edi
+; CHECK-NEXT: popl %ebx
+; CHECK-NEXT: popl %ebp
+; CHECK-NEXT: retl
%val = load <16 x bfloat>, ptr %ptr
%bf16 = call <16 x bfloat> @returns_v16bf16(<16 x bfloat> %val)
store <16 x bfloat> %bf16, ptr %ptr
@@ -1241,3 +358,6 @@ define void @call_ret_v16bf16(ptr %ptr) #0 {
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NOSSE: {{.*}}
+; SSE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/bfloat-calling-conv.ll b/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
index 25f4c9fcfee51..0b01b00b3caa2 100644
--- a/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
@@ -13,10 +13,8 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %rax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: popq %rax
+; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_bf16:
@@ -25,8 +23,8 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
;
; FAST_ISEL_AVX512BF16-LABEL: return_arg_bf16:
; FAST_ISEL_AVX512BF16: # %bb.0:
-; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: return_arg_bf16:
@@ -35,8 +33,8 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
;
; FAST_ISEL_AVXNECONVERT-LABEL: return_arg_bf16:
; FAST_ISEL_AVXNECONVERT: # %bb.0:
-; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
+; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: retq
ret bfloat %x
}
@@ -48,20 +46,11 @@ define <2 x bfloat> @return_arg_v2bf16(<2 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v2bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $40, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
-; FAST_ISEL_SSE2-NEXT: addq $40, %rsp
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v2bf16:
@@ -89,29 +78,14 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v3bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; FAST_ISEL_SSE2-NEXT: movaps %xmm1, %xmm0
-; FAST_ISEL_SSE2-NEXT: addq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %edx, %xmm0
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %ecx, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v3bf16:
@@ -120,17 +94,11 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_AVX512BF16-LABEL: return_arg_v3bf16:
; FAST_ISEL_AVX512BF16: # %bb.0:
-; FAST_ISEL_AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm2
; FAST_ISEL_AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm0, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; FAST_ISEL_AVX512BF16-NEXT: vpextrw $1, %xmm0, %ecx
+; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm0, %edx
+; FAST_ISEL_AVX512BF16-NEXT: vmovd %edx, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: retq
;
@@ -140,23 +108,18 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_AVXNECONVERT-LABEL: return_arg_v3bf16:
; FAST_ISEL_AVXNECONVERT: # %bb.0:
-; FAST_ISEL_AVXNECONVERT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm2
; FAST_ISEL_AVXNECONVERT-NEXT: vpextrw $2, %xmm0, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm0, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm2, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm1, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovq %xmm1, %rax
+; FAST_ISEL_AVXNECONVERT-NEXT: vpextrw $1, %xmm0, %ecx
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm0, %edx
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %edx, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovq %xmm0, %rax
; FAST_ISEL_AVXNECONVERT-NEXT: movl %eax, %ecx
; FAST_ISEL_AVXNECONVERT-NEXT: shrl $16, %ecx
-; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %ecx, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpbroadcastw %xmm1, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
; FAST_ISEL_AVXNECONVERT-NEXT: shrq $32, %rax
; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm1
; FAST_ISEL_AVXNECONVERT-NEXT: vpbroadcastw %xmm1, %xmm1
@@ -172,37 +135,17 @@ define <4 x bfloat> @return_arg_v4bf16(<4 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v4bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $72, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %esi
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %esi, %xmm0
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %edx, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %ecx, %xmm1
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; FAST_ISEL_SSE2-NEXT: addq $72, %rsp
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v4bf16:
@@ -230,84 +173,33 @@ define <8 x bfloat> @return_arg_v8bf16(<8 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v8bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $120, %rsp
-; FAST_ISEL_SSE2-NEXT: pxor %xmm1, %xmm1
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm0, %esi
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %edi
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %r8d
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %r9d
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %r10d
+; FAST_ISEL_SSE2-NEXT: shll $16, %r9d
+; FAST_ISEL_SSE2-NEXT: movzwl %r10w, %r10d
+; FAST_ISEL_SSE2-NEXT: orl %r9d, %r10d
+; FAST_ISEL_SSE2-NEXT: shll $16, %edi
+; FAST_ISEL_SSE2-NEXT: movzwl %r8w, %r8d
+; FAST_ISEL_SSE2-NEXT: orl %edi, %r8d
+; FAST_ISEL_SSE2-NEXT: shlq $32, %r8
+; FAST_ISEL_SSE2-NEXT: orq %r10, %r8
+; FAST_ISEL_SSE2-NEXT: movq %r8, %xmm0
+; FAST_ISEL_SSE2-NEXT: shll $16, %edx
+; FAST_ISEL_SSE2-NEXT: movzwl %si, %esi
+; FAST_ISEL_SSE2-NEXT: orl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rcx
+; FAST_ISEL_SSE2-NEXT: orq %rsi, %rcx
+; FAST_ISEL_SSE2-NEXT: movq %rcx, %xmm1
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; FAST_ISEL_SSE2-NEXT: addq $120, %rsp
-; FAST_ISEL_SSE2-NEXT: popq %rbx
-; FAST_ISEL_SSE2-NEXT: popq %r14
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v8bf16:
@@ -336,160 +228,62 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v16bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $248, %rsp
-; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
-; FAST_ISEL_SSE2-NEXT: pxor %xmm2, %xmm2
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm6
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm8
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm7
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm9
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm9
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm10
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: addq $248, %rsp
-; FAST_ISEL_SSE2-NEXT: popq %rbx
-; FAST_ISEL_SSE2-NEXT: popq %r14
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rcx
+; FAST_ISEL_SSE2-NEXT: orq %rdx, %rcx
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: movq %rcx, %xmm0
+; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm2, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %edx
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm2, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rcx, %rdx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm3
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rcx
+; FAST_ISEL_SSE2-NEXT: orq %rdx, %rcx
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: movq %rcx, %xmm2
+; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %edx
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rcx, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, %xmm1
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v16bf16:
@@ -529,9 +323,7 @@ define bfloat @call_ret_bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movzwl (%rdi), %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
; FAST_ISEL_SSE2-NEXT: callq returns_bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_bf16:
@@ -544,9 +336,7 @@ define bfloat @call_ret_bf16(ptr %ptr) #0 {
; FAST_ISEL_AVX512BF16: # %bb.0:
; FAST_ISEL_AVX512BF16-NEXT: pushq %rax
; FAST_ISEL_AVX512BF16-NEXT: movzwl (%rdi), %eax
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: callq returns_bf16 at PLT
;
; AVXNECONVERT-LABEL: call_ret_bf16:
@@ -559,9 +349,7 @@ define bfloat @call_ret_bf16(ptr %ptr) #0 {
; FAST_ISEL_AVXNECONVERT: # %bb.0:
; FAST_ISEL_AVXNECONVERT-NEXT: pushq %rax
; FAST_ISEL_AVXNECONVERT-NEXT: movzwl (%rdi), %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: callq returns_bf16 at PLT
%val = load bfloat, ptr %ptr
call bfloat @returns_bf16(bfloat %val)
@@ -578,22 +366,15 @@ define <2 x bfloat> @call_ret_v2bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v2bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movl (%rdi), %eax
; FAST_ISEL_SSE2-NEXT: movl %eax, (%rsp)
; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; FAST_ISEL_SSE2-NEXT: callq returns_v2bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v2bf16:
@@ -636,30 +417,16 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v3bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movq (%rdi), %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm0
; FAST_ISEL_SSE2-NEXT: shrq $32, %rax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: psrld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; FAST_ISEL_SSE2-NEXT: movaps %xmm1, %xmm0
+; FAST_ISEL_SSE2-NEXT: shrl $16, %ecx
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %ecx, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; FAST_ISEL_SSE2-NEXT: callq returns_v3bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v3bf16:
@@ -672,20 +439,13 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
; FAST_ISEL_AVX512BF16: # %bb.0:
; FAST_ISEL_AVX512BF16-NEXT: pushq %rax
; FAST_ISEL_AVX512BF16-NEXT: movq (%rdi), %rax
+; FAST_ISEL_AVX512BF16-NEXT: movq %rax, %rcx
+; FAST_ISEL_AVX512BF16-NEXT: shrq $32, %rcx
+; FAST_ISEL_AVX512BF16-NEXT: movl %eax, %edx
+; FAST_ISEL_AVX512BF16-NEXT: shrl $16, %edx
; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; FAST_ISEL_AVX512BF16-NEXT: vmovq %rax, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: shrq $32, %rax
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm2
-; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm2, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $1, %edx, %xmm0, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $2, %ecx, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: callq returns_v3bf16 at PLT
;
; AVXNECONVERT-LABEL: call_ret_v3bf16:
@@ -700,25 +460,18 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
; FAST_ISEL_AVXNECONVERT: # %bb.0:
; FAST_ISEL_AVXNECONVERT-NEXT: pushq %rax
; FAST_ISEL_AVXNECONVERT-NEXT: movq (%rdi), %rax
+; FAST_ISEL_AVXNECONVERT-NEXT: movl %eax, %ecx
; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovq %rax, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
; FAST_ISEL_AVXNECONVERT-NEXT: shrq $32, %rax
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm2
-; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm2, %xmm2
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm2, %xmm2
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm2, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm1, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; FAST_ISEL_AVXNECONVERT-NEXT: shrl $16, %ecx
+; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: vmovq %xmm0, %rax
; FAST_ISEL_AVXNECONVERT-NEXT: movl %eax, %ecx
; FAST_ISEL_AVXNECONVERT-NEXT: shrl $16, %ecx
-; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %ecx, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpbroadcastw %xmm1, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
; FAST_ISEL_AVXNECONVERT-NEXT: shrq $32, %rax
; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm1
; FAST_ISEL_AVXNECONVERT-NEXT: vpbroadcastw %xmm1, %xmm1
@@ -739,38 +492,20 @@ define <4 x bfloat> @call_ret_v4bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v4bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $88, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movq (%rdi), %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, {{[0-9]+}}(%rsp)
-; FAST_ISEL_SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; FAST_ISEL_SSE2-NEXT: movq %rax, (%rsp)
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %esi
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %esi, %xmm0
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %edx, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %ecx, %xmm1
+; FAST_ISEL_SSE2-NEXT: pinsrw $0, %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; FAST_ISEL_SSE2-NEXT: callq returns_v4bf16 at PLT
;
@@ -812,81 +547,34 @@ define <8 x bfloat> @call_ret_v8bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v8bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $120, %rsp
+; FAST_ISEL_SSE2-NEXT: pushq %rax
; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm0
-; FAST_ISEL_SSE2-NEXT: pxor %xmm1, %xmm1
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm0, %esi
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %edi
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %r8d
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %r9d
+; FAST_ISEL_SSE2-NEXT: movd %xmm0, %r10d
+; FAST_ISEL_SSE2-NEXT: shll $16, %r9d
+; FAST_ISEL_SSE2-NEXT: movzwl %r10w, %r10d
+; FAST_ISEL_SSE2-NEXT: orl %r9d, %r10d
+; FAST_ISEL_SSE2-NEXT: shll $16, %edi
+; FAST_ISEL_SSE2-NEXT: movzwl %r8w, %r8d
+; FAST_ISEL_SSE2-NEXT: orl %edi, %r8d
+; FAST_ISEL_SSE2-NEXT: shlq $32, %r8
+; FAST_ISEL_SSE2-NEXT: orq %r10, %r8
+; FAST_ISEL_SSE2-NEXT: movq %r8, %xmm0
+; FAST_ISEL_SSE2-NEXT: shll $16, %edx
+; FAST_ISEL_SSE2-NEXT: movzwl %si, %esi
+; FAST_ISEL_SSE2-NEXT: orl %edx, %esi
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rcx
+; FAST_ISEL_SSE2-NEXT: orq %rsi, %rcx
+; FAST_ISEL_SSE2-NEXT: movq %rcx, %xmm1
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; FAST_ISEL_SSE2-NEXT: callq returns_v8bf16 at PLT
;
@@ -929,159 +617,63 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v16bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: pushq %r14
-; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $248, %rsp
-; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa 16(%rdi), %xmm1
-; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
-; FAST_ISEL_SSE2-NEXT: pxor %xmm2, %xmm2
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
+; FAST_ISEL_SSE2-NEXT: pushq %rax
+; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa 16(%rdi), %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm6
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm8
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm7
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm9
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm9
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm10
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
-; FAST_ISEL_SSE2-NEXT: orl %ebx, %eax
-; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
-; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
-; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: movd %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rcx
+; FAST_ISEL_SSE2-NEXT: orq %rdx, %rcx
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: movq %rcx, %xmm0
+; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %edx
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm2, %eax
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rcx, %rdx
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm2, %ecx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm3
+; FAST_ISEL_SSE2-NEXT: movd %xmm2, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %ecx
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %ecx, %edx
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm2, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm2, %eax
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rcx
+; FAST_ISEL_SSE2-NEXT: orq %rdx, %rcx
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm2, %edx
+; FAST_ISEL_SSE2-NEXT: movq %rcx, %xmm1
+; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm2, %ecx
+; FAST_ISEL_SSE2-NEXT: shll $16, %edx
+; FAST_ISEL_SSE2-NEXT: movzwl %cx, %ecx
+; FAST_ISEL_SSE2-NEXT: orl %edx, %ecx
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm2, %edx
+; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: movzwl %dx, %edx
+; FAST_ISEL_SSE2-NEXT: orl %eax, %edx
+; FAST_ISEL_SSE2-NEXT: shlq $32, %rdx
+; FAST_ISEL_SSE2-NEXT: orq %rcx, %rdx
+; FAST_ISEL_SSE2-NEXT: movq %rdx, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; FAST_ISEL_SSE2-NEXT: callq returns_v16bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v16bf16:
diff --git a/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll b/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll
index 9236951501a5f..400b828c02c4f 100644
--- a/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll
+++ b/llvm/test/CodeGen/X86/bfloat-int64-cvt-i686.ll
@@ -18,10 +18,6 @@ define bfloat @sitofp_i64_to_bf16(i64 %a) nounwind {
; X87-NEXT: fildll {{[0-9]+}}(%esp)
; X87-NEXT: fstps (%esp)
; X87-NEXT: calll __truncsfbf2
-; X87-NEXT: # kill: def $ax killed $ax def $eax
-; X87-NEXT: shll $16, %eax
-; X87-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT: flds {{[0-9]+}}(%esp)
; X87-NEXT: addl $28, %esp
; X87-NEXT: retl
;
@@ -72,10 +68,6 @@ define bfloat @sitofp_load_i64_to_bf16(ptr %p) nounwind {
; X87-NEXT: fildll {{[0-9]+}}(%esp)
; X87-NEXT: fstps (%esp)
; X87-NEXT: calll __truncsfbf2
-; X87-NEXT: # kill: def $ax killed $ax def $eax
-; X87-NEXT: shll $16, %eax
-; X87-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT: flds {{[0-9]+}}(%esp)
; X87-NEXT: addl $28, %esp
; X87-NEXT: retl
;
@@ -130,10 +122,6 @@ define bfloat @uitofp_i64_to_bf16(i64 %a) nounwind {
; X87-NEXT: fadds {{\.?LCPI[0-9]+_[0-9]+}}(,%ecx,4)
; X87-NEXT: fstps (%esp)
; X87-NEXT: calll __truncsfbf2
-; X87-NEXT: # kill: def $ax killed $ax def $eax
-; X87-NEXT: shll $16, %eax
-; X87-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT: flds {{[0-9]+}}(%esp)
; X87-NEXT: addl $28, %esp
; X87-NEXT: retl
;
@@ -199,10 +187,6 @@ define bfloat @uitofp_load_i64_to_bf16(ptr %p) nounwind {
; X87-NEXT: fadds {{\.?LCPI[0-9]+_[0-9]+}}(,%eax,4)
; X87-NEXT: fstps (%esp)
; X87-NEXT: calll __truncsfbf2
-; X87-NEXT: # kill: def $ax killed $ax def $eax
-; X87-NEXT: shll $16, %eax
-; X87-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT: flds {{[0-9]+}}(%esp)
; X87-NEXT: addl $28, %esp
; X87-NEXT: retl
;
@@ -271,11 +255,6 @@ define bfloat @strict_sitofp_load_i64_to_bf16(ptr %p) strictfp nounwind {
; X87-NEXT: fstps (%esp)
; X87-NEXT: wait
; X87-NEXT: calll __truncsfbf2
-; X87-NEXT: # kill: def $ax killed $ax def $eax
-; X87-NEXT: shll $16, %eax
-; X87-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT: flds {{[0-9]+}}(%esp)
-; X87-NEXT: wait
; X87-NEXT: addl $28, %esp
; X87-NEXT: retl
;
@@ -340,11 +319,6 @@ define bfloat @strict_uitofp_i64_to_bf16(i64 %a) strictfp nounwind {
; X87-NEXT: fstps (%esp)
; X87-NEXT: wait
; X87-NEXT: calll __truncsfbf2
-; X87-NEXT: # kill: def $ax killed $ax def $eax
-; X87-NEXT: shll $16, %eax
-; X87-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X87-NEXT: flds {{[0-9]+}}(%esp)
-; X87-NEXT: wait
; X87-NEXT: addl $28, %esp
; X87-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 1d5e65afec89c..e86bd7e2fb437 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -796,208 +796,185 @@ define <32 x bfloat> @pr63017_2() nounwind {
;
; SSE2-LABEL: pr63017_2:
; SSE2: # %bb.0:
+; SSE2-NEXT: pushq %rbp
+; SSE2-NEXT: pushq %r15
+; SSE2-NEXT: pushq %r14
+; SSE2-NEXT: pushq %r13
+; SSE2-NEXT: pushq %r12
+; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: testb %al, %al
; SSE2-NEXT: jne .LBB16_1
; SSE2-NEXT: # %bb.2: # %cond.load
-; SSE2-NEXT: movzwl (%rax), %eax
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movzwl (%rax), %edx
+; SSE2-NEXT: movl %edx, %edi
+; SSE2-NEXT: movw %dx, %ax
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movw %dx, %cx
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r14d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movw %dx, %si
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r8d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r12d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %ebx
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r10d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r13d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r15d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %ebp
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r11d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %edx, %r9d
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; SSE2-NEXT: movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: movw %dx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; SSE2-NEXT: jmp .LBB16_3
; SSE2-NEXT: .LBB16_1:
-; SSE2-NEXT: movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; SSE2-NEXT: movw $-16512, %dx # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %di # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %ax # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r14w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %cx # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r8w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r12w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %bx # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r10w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r13w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r15w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %bp # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r11w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %r9w # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %si # imm = 0xBF80
+; SSE2-NEXT: movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %si # imm = 0xBF80
+; SSE2-NEXT: movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: movw $-16512, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Folded Spill
+; SSE2-NEXT: # imm = 0xBF80
+; SSE2-NEXT: movw $-16512, %si # imm = 0xBF80
; SSE2-NEXT: .LBB16_3:
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $88, %rsp
-; SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
+; SSE2-NEXT: movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT: shll $16, %edi
+; SSE2-NEXT: movzwl %dx, %esi
+; SSE2-NEXT: orl %edi, %esi
+; SSE2-NEXT: shll $16, %eax
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 2-byte Folded Reload
+; SSE2-NEXT: orl %eax, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rsi, %rdx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: shll $16, %r14d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; SSE2-NEXT: orl %r14d, %eax
+; SSE2-NEXT: shll $16, %ecx
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 2-byte Folded Reload
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: shlq $32, %rdx
+; SSE2-NEXT: orq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm4
+; SSE2-NEXT: shll $16, %r8d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; SSE2-NEXT: orl %r8d, %eax
+; SSE2-NEXT: shll $16, %r12d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 2-byte Folded Reload
+; SSE2-NEXT: orl %r12d, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm1
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
-; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
; SSE2-NEXT: orl %ebx, %eax
-; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
-; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT: addq $88, %rsp
+; SSE2-NEXT: shll $16, %r10d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 2-byte Folded Reload
+; SSE2-NEXT: orl %r10d, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm5
+; SSE2-NEXT: shll $16, %r13d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; SSE2-NEXT: orl %r13d, %eax
+; SSE2-NEXT: shll $16, %r15d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 2-byte Folded Reload
+; SSE2-NEXT: orl %r15d, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm2
+; SSE2-NEXT: shll $16, %ebp
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; SSE2-NEXT: orl %ebp, %eax
+; SSE2-NEXT: shll $16, %r11d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 2-byte Folded Reload
+; SSE2-NEXT: orl %r11d, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm6
+; SSE2-NEXT: shll $16, %r9d
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; SSE2-NEXT: orl %r9d, %eax
+; SSE2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Reload
+; SSE2-NEXT: shll $16, %edx
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 2-byte Folded Reload
+; SSE2-NEXT: orl %edx, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm3
+; SSE2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
+; SSE2-NEXT: shll $16, %ecx
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 2-byte Folded Reload
+; SSE2-NEXT: orl %ecx, %eax
+; SSE2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %edx # 4-byte Reload
+; SSE2-NEXT: shll $16, %edx
+; SSE2-NEXT: movzwl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 2-byte Folded Reload
+; SSE2-NEXT: orl %edx, %ecx
+; SSE2-NEXT: shlq $32, %rcx
+; SSE2-NEXT: orq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm7
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: popq %r12
+; SSE2-NEXT: popq %r13
; SSE2-NEXT: popq %r14
+; SSE2-NEXT: popq %r15
+; SSE2-NEXT: popq %rbp
; SSE2-NEXT: retq
;
; AVX512BF16-LABEL: pr63017_2:
@@ -2078,3 +2055,64 @@ define bfloat @PR115710(fp128 %0) nounwind {
%2 = fptrunc fp128 %0 to bfloat
ret bfloat %2
}
+
+define void @test_bitcast_bfloattoi16_in_branch(bfloat %a, i1 %c, ptr %p) nounwind {
+; X86-LABEL: test_bitcast_bfloattoi16_in_branch:
+; X86: # %bb.0: # %entry
+; X86-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB35_2
+; X86-NEXT: # %bb.1: # %bitcast
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movw %cx, (%eax)
+; X86-NEXT: .LBB35_2: # %exit
+; X86-NEXT: retl
+;
+; SSE2-LABEL: test_bitcast_bfloattoi16_in_branch:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: je .LBB35_2
+; SSE2-NEXT: # %bb.1: # %bitcast
+; SSE2-NEXT: pextrw $0, %xmm0, %eax
+; SSE2-NEXT: movw %ax, (%rsi)
+; SSE2-NEXT: .LBB35_2: # %exit
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: test_bitcast_bfloattoi16_in_branch:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: testb $1, %dil
+; AVX512BF16-NEXT: je .LBB35_2
+; AVX512BF16-NEXT: # %bb.1: # %bitcast
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: movw %ax, (%rsi)
+; AVX512BF16-NEXT: .LBB35_2: # %exit
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: test_bitcast_bfloattoi16_in_branch:
+; AVX512FP16: # %bb.0: # %entry
+; AVX512FP16-NEXT: vmovw %xmm0, %eax
+; AVX512FP16-NEXT: testb $1, %dil
+; AVX512FP16-NEXT: je .LBB35_2
+; AVX512FP16-NEXT: # %bb.1: # %bitcast
+; AVX512FP16-NEXT: movw %ax, (%rsi)
+; AVX512FP16-NEXT: .LBB35_2: # %exit
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: test_bitcast_bfloattoi16_in_branch:
+; AVXNC: # %bb.0: # %entry
+; AVXNC-NEXT: testb $1, %dil
+; AVXNC-NEXT: je .LBB35_2
+; AVXNC-NEXT: # %bb.1: # %bitcast
+; AVXNC-NEXT: vpextrw $0, %xmm0, %eax
+; AVXNC-NEXT: movw %ax, (%rsi)
+; AVXNC-NEXT: .LBB35_2: # %exit
+; AVXNC-NEXT: retq
+entry:
+ br i1 %c, label %bitcast, label %exit
+bitcast:
+ %b = bitcast bfloat %a to i16
+ store i16 %b, ptr %p
+ br label %exit
+exit:
+ ret void
+}
>From caf036cf5050d1926cbf5f57499eaf31b69c5f7d Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 20 Sep 2026 13:31:33 +0200
Subject: [PATCH 2/5] update amdgpu tests
---
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 41685 +++++++---------
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 2916 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll | 92 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 5899 +--
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 577 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 375 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 14737 +++---
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 1487 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 1715 +-
.../atomic_optimizations_global_pointer.ll | 76 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 927 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 431 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 431 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 590 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 482 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 482 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 436 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 1352 +-
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 932 +-
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 932 +-
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 824 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 326 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 358 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 358 +-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 326 +-
llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll | 25 +-
26 files changed, 34831 insertions(+), 43940 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 47468f3c7c4cf..428959a141f38 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -21956,596 +21956,223 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr62
; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr60
; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr59
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr58
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB16_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v62
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v53, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v55, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v42, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v44, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v47, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v62
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v13
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v12
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v11
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v10
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v9
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v8
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v7
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v2
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v1
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB16_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB16_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v31, vcc, 3, v62
-; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v30
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_add_i32_e32 v29, vcc, 3, v29
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_add_i32_e32 v28, vcc, 3, v28
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_add_i32_e32 v27, vcc, 3, v27
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_add_i32_e32 v26, vcc, 3, v26
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_add_i32_e32 v23, vcc, 3, v23
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT: v_add_i32_e32 v17, vcc, 3, v17
; SI-NEXT: v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT: v_add_i32_e32 v18, vcc, 3, v18
+; SI-NEXT: v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT: v_add_i32_e32 v20, vcc, 3, v20
+; SI-NEXT: v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT: v_add_i32_e32 v22, vcc, 3, v22
+; SI-NEXT: v_add_i32_e32 v25, vcc, 3, v25
+; SI-NEXT: v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT: v_add_i32_e32 v27, vcc, 3, v27
+; SI-NEXT: v_add_i32_e32 v26, vcc, 3, v26
+; SI-NEXT: v_add_i32_e32 v29, vcc, 3, v29
+; SI-NEXT: v_add_i32_e32 v28, vcc, 3, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v31, vcc, 3, v31
+; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v30
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v53, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v55, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v42, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v44, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v47, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v13
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v12
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v11
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v10
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v9
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v8
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v7
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v2
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v1
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB16_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v60
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v58
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v56
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v46
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v44
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v42
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v40
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; SI-NEXT: v_or_b32_e32 v0, v0, v47
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v63
+; SI-NEXT: v_or_b32_e32 v2, v2, v44
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v62
+; SI-NEXT: v_or_b32_e32 v4, v4, v42
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v61
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_or_b32_e32 v1, v1, v47
+; SI-NEXT: v_or_b32_e32 v3, v3, v44
+; SI-NEXT: v_or_b32_e32 v5, v5, v42
+; SI-NEXT: v_or_b32_e32 v6, v6, v55
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v60
+; SI-NEXT: v_or_b32_e32 v8, v8, v53
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v59
+; SI-NEXT: v_or_b32_e32 v10, v10, v50
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v58
+; SI-NEXT: v_or_b32_e32 v12, v12, v49
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v57
+; SI-NEXT: v_or_b32_e32 v14, v14, v48
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v56
+; SI-NEXT: v_or_b32_e32 v16, v16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v46
+; SI-NEXT: v_or_b32_e32 v18, v18, v38
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v45
+; SI-NEXT: v_or_b32_e32 v20, v20, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v43
+; SI-NEXT: v_or_b32_e32 v22, v22, v36
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v41
+; SI-NEXT: v_or_b32_e32 v24, v24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v40
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v63
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@@ -22562,8 +22189,45 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v26, v26, v34
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v54
+; SI-NEXT: v_or_b32_e32 v28, v28, v33
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v52
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v51
+; SI-NEXT: v_or_b32_e32 v7, v7, v55
+; SI-NEXT: v_or_b32_e32 v9, v9, v53
+; SI-NEXT: v_or_b32_e32 v11, v11, v50
+; SI-NEXT: v_or_b32_e32 v13, v13, v49
+; SI-NEXT: v_or_b32_e32 v15, v15, v48
+; SI-NEXT: v_or_b32_e32 v17, v17, v39
+; SI-NEXT: v_or_b32_e32 v19, v19, v38
+; SI-NEXT: v_or_b32_e32 v21, v21, v37
+; SI-NEXT: v_or_b32_e32 v23, v23, v36
+; SI-NEXT: v_or_b32_e32 v25, v25, v35
+; SI-NEXT: v_or_b32_e32 v27, v27, v34
+; SI-NEXT: v_or_b32_e32 v29, v29, v33
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -22740,574 +22404,340 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_writelane_b32 v33, s34, 0
-; SI-NEXT: v_writelane_b32 v33, s35, 1
-; SI-NEXT: v_writelane_b32 v33, s36, 2
-; SI-NEXT: v_writelane_b32 v33, s37, 3
-; SI-NEXT: v_writelane_b32 v33, s38, 4
-; SI-NEXT: v_writelane_b32 v33, s39, 5
-; SI-NEXT: v_writelane_b32 v33, s48, 6
-; SI-NEXT: v_writelane_b32 v33, s49, 7
-; SI-NEXT: v_writelane_b32 v33, s50, 8
-; SI-NEXT: v_writelane_b32 v33, s51, 9
-; SI-NEXT: v_writelane_b32 v33, s52, 10
-; SI-NEXT: v_writelane_b32 v33, s53, 11
-; SI-NEXT: v_writelane_b32 v33, s54, 12
-; SI-NEXT: v_writelane_b32 v33, s55, 13
-; SI-NEXT: v_writelane_b32 v33, s64, 14
-; SI-NEXT: v_writelane_b32 v33, s65, 15
-; SI-NEXT: v_writelane_b32 v33, s66, 16
-; SI-NEXT: v_writelane_b32 v33, s67, 17
-; SI-NEXT: v_writelane_b32 v33, s68, 18
-; SI-NEXT: v_writelane_b32 v33, s69, 19
-; SI-NEXT: v_writelane_b32 v33, s70, 20
-; SI-NEXT: v_writelane_b32 v33, s71, 21
-; SI-NEXT: v_writelane_b32 v33, s80, 22
-; SI-NEXT: v_writelane_b32 v33, s81, 23
-; SI-NEXT: v_writelane_b32 v33, s82, 24
-; SI-NEXT: v_writelane_b32 v33, s83, 25
-; SI-NEXT: v_writelane_b32 v33, s84, 26
-; SI-NEXT: v_writelane_b32 v33, s85, 27
-; SI-NEXT: v_writelane_b32 v33, s86, 28
-; SI-NEXT: v_writelane_b32 v33, s87, 29
-; SI-NEXT: v_writelane_b32 v33, s96, 30
-; SI-NEXT: v_writelane_b32 v33, s97, 31
-; SI-NEXT: v_writelane_b32 v33, s98, 32
-; SI-NEXT: v_writelane_b32 v33, s30, 33
-; SI-NEXT: v_writelane_b32 v33, s31, 34
-; SI-NEXT: v_readfirstlane_b32 s5, v18
-; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: v_readfirstlane_b32 s30, v16
-; SI-NEXT: v_readfirstlane_b32 s31, v15
-; SI-NEXT: v_readfirstlane_b32 s34, v14
-; SI-NEXT: v_readfirstlane_b32 s35, v13
-; SI-NEXT: v_readfirstlane_b32 s36, v12
-; SI-NEXT: v_readfirstlane_b32 s37, v11
-; SI-NEXT: v_readfirstlane_b32 s38, v10
-; SI-NEXT: v_readfirstlane_b32 s39, v9
-; SI-NEXT: v_readfirstlane_b32 s48, v8
-; SI-NEXT: v_readfirstlane_b32 s49, v7
-; SI-NEXT: v_readfirstlane_b32 s50, v6
-; SI-NEXT: v_readfirstlane_b32 s51, v5
-; SI-NEXT: v_readfirstlane_b32 s52, v4
-; SI-NEXT: v_readfirstlane_b32 s53, v3
-; SI-NEXT: v_readfirstlane_b32 s54, v2
-; SI-NEXT: v_readfirstlane_b32 s55, v1
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_readfirstlane_b32 s64, v0
-; SI-NEXT: ; implicit-def: $vgpr34 : SGPR spill to VGPR lane
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v34, s4, 0
+; SI-NEXT: v_writelane_b32 v32, s34, 0
+; SI-NEXT: v_writelane_b32 v32, s35, 1
+; SI-NEXT: v_writelane_b32 v32, s36, 2
+; SI-NEXT: v_writelane_b32 v32, s37, 3
+; SI-NEXT: v_writelane_b32 v32, s38, 4
+; SI-NEXT: v_writelane_b32 v32, s39, 5
+; SI-NEXT: v_writelane_b32 v32, s48, 6
+; SI-NEXT: v_writelane_b32 v32, s49, 7
+; SI-NEXT: v_writelane_b32 v32, s50, 8
+; SI-NEXT: v_writelane_b32 v32, s51, 9
+; SI-NEXT: v_writelane_b32 v32, s52, 10
+; SI-NEXT: v_writelane_b32 v32, s53, 11
+; SI-NEXT: v_writelane_b32 v32, s54, 12
+; SI-NEXT: v_writelane_b32 v32, s55, 13
+; SI-NEXT: v_writelane_b32 v32, s64, 14
+; SI-NEXT: v_writelane_b32 v32, s65, 15
+; SI-NEXT: v_writelane_b32 v32, s30, 16
+; SI-NEXT: v_writelane_b32 v32, s31, 17
+; SI-NEXT: v_readfirstlane_b32 s44, v18
+; SI-NEXT: v_readfirstlane_b32 s5, v17
+; SI-NEXT: v_readfirstlane_b32 s4, v16
+; SI-NEXT: v_readfirstlane_b32 s7, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: v_readfirstlane_b32 s9, v13
+; SI-NEXT: v_readfirstlane_b32 s8, v12
+; SI-NEXT: v_readfirstlane_b32 s11, v11
+; SI-NEXT: v_readfirstlane_b32 s10, v10
+; SI-NEXT: v_readfirstlane_b32 s13, v9
+; SI-NEXT: v_readfirstlane_b32 s12, v8
+; SI-NEXT: v_readfirstlane_b32 s15, v7
+; SI-NEXT: v_readfirstlane_b32 s14, v6
+; SI-NEXT: v_readfirstlane_b32 s41, v5
+; SI-NEXT: v_readfirstlane_b32 s40, v4
+; SI-NEXT: v_readfirstlane_b32 s43, v3
+; SI-NEXT: v_readfirstlane_b32 s42, v2
+; SI-NEXT: v_readfirstlane_b32 s45, v1
+; SI-NEXT: s_cmp_lg_u32 s44, 0
+; SI-NEXT: v_readfirstlane_b32 s44, v0
; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s4, 16
-; SI-NEXT: s_and_b32 s4, s50, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 2
-; SI-NEXT: s_lshl_b32 s4, s50, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 1
-; SI-NEXT: s_and_b32 s4, s51, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 4
-; SI-NEXT: s_lshl_b32 s4, s51, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 3
-; SI-NEXT: s_and_b32 s4, s52, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 6
-; SI-NEXT: s_lshl_b32 s4, s52, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 5
-; SI-NEXT: s_and_b32 s4, s53, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 8
-; SI-NEXT: s_lshl_b32 s4, s53, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 7
-; SI-NEXT: s_and_b32 s4, s55, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 10
-; SI-NEXT: s_lshl_b32 s4, s55, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 9
-; SI-NEXT: s_and_b32 s4, s64, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 12
-; SI-NEXT: s_lshl_b32 s4, s64, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 11
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 14
-; SI-NEXT: s_lshl_b32 s4, s29, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 13
-; SI-NEXT: s_and_b32 s4, s28, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 16
-; SI-NEXT: s_lshl_b32 s4, s28, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 15
-; SI-NEXT: s_and_b32 s4, s27, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 18
-; SI-NEXT: s_lshl_b32 s4, s27, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 17
-; SI-NEXT: s_and_b32 s4, s26, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 20
-; SI-NEXT: s_lshl_b32 s4, s26, 16
-; SI-NEXT: s_and_b32 s10, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s30, 16
-; SI-NEXT: s_and_b32 s12, s31, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s31, 16
-; SI-NEXT: s_and_b32 s14, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s34, 16
-; SI-NEXT: s_and_b32 s40, s35, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s35, 16
-; SI-NEXT: s_and_b32 s42, s36, 0xffff0000
-; SI-NEXT: s_lshl_b32 s41, s36, 16
-; SI-NEXT: s_and_b32 s44, s37, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s37, 16
-; SI-NEXT: s_and_b32 s46, s38, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s38, 16
-; SI-NEXT: s_and_b32 s56, s39, 0xffff0000
-; SI-NEXT: s_lshl_b32 s47, s39, 16
-; SI-NEXT: s_and_b32 s58, s48, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s48, 16
-; SI-NEXT: s_and_b32 s60, s49, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s49, 16
-; SI-NEXT: s_and_b32 s78, s54, 0xffff0000
-; SI-NEXT: s_lshl_b32 s77, s54, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 19
-; SI-NEXT: s_and_b32 s62, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s61, s25, 16
-; SI-NEXT: s_and_b32 s72, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s63, s24, 16
-; SI-NEXT: s_and_b32 s74, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s73, s23, 16
-; SI-NEXT: s_and_b32 s76, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s75, s22, 16
-; SI-NEXT: s_and_b32 s88, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s79, s21, 16
-; SI-NEXT: s_and_b32 s90, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s89, s20, 16
-; SI-NEXT: s_and_b32 s92, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s91, s19, 16
-; SI-NEXT: s_and_b32 s94, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s93, s18, 16
-; SI-NEXT: s_and_b32 vcc_lo, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s95, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_hi, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s56, s5, 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s58, s45, 16
+; SI-NEXT: s_mov_b32 s47, s56
+; SI-NEXT: s_lshr_b64 s[56:57], s[6:7], 16
+; SI-NEXT: s_lshr_b32 s62, s19, 16
+; SI-NEXT: s_lshr_b32 s63, s17, 16
+; SI-NEXT: s_mov_b32 s57, s58
+; SI-NEXT: s_lshr_b64 s[60:61], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
+; SI-NEXT: s_lshr_b32 s38, s7, 16
+; SI-NEXT: s_lshr_b32 s39, s9, 16
+; SI-NEXT: s_lshr_b32 s48, s11, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s50, s15, 16
+; SI-NEXT: s_lshr_b32 s51, s41, 16
+; SI-NEXT: s_lshr_b32 s52, s43, 16
+; SI-NEXT: s_lshr_b32 s53, s29, 16
+; SI-NEXT: s_lshr_b32 s54, s27, 16
+; SI-NEXT: s_lshr_b32 s55, s25, 16
+; SI-NEXT: s_lshr_b32 s64, s23, 16
+; SI-NEXT: s_lshr_b32 s65, s21, 16
+; SI-NEXT: s_mov_b32 s61, s63
+; SI-NEXT: s_mov_b32 s59, s62
+; SI-NEXT: s_lshr_b64 s[62:63], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
; SI-NEXT: s_branch .LBB17_3
; SI-NEXT: .LBB17_2:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr65
; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr64
; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr55
; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr54
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr52
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr51
; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr49
; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr45
; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: .LBB17_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_mov_b32 s65, s63
-; SI-NEXT: s_mov_b32 s66, s72
-; SI-NEXT: s_mov_b32 s67, s73
-; SI-NEXT: s_mov_b32 s68, s74
-; SI-NEXT: s_mov_b32 s69, s75
-; SI-NEXT: s_mov_b32 s70, s76
-; SI-NEXT: s_mov_b32 s71, s79
-; SI-NEXT: s_mov_b32 s80, s88
-; SI-NEXT: s_mov_b32 s81, s89
-; SI-NEXT: s_mov_b32 s82, s90
-; SI-NEXT: s_mov_b32 s83, s91
-; SI-NEXT: s_mov_b32 s84, s92
-; SI-NEXT: s_mov_b32 s85, s93
-; SI-NEXT: s_mov_b32 s86, s94
-; SI-NEXT: s_mov_b32 s87, s95
-; SI-NEXT: s_mov_b32 s96, vcc_lo
-; SI-NEXT: s_mov_b32 vcc_lo, s61
-; SI-NEXT: s_mov_b32 s97, vcc_hi
-; SI-NEXT: s_mov_b32 vcc_hi, s62
-; SI-NEXT: s_mov_b32 s98, s6
-; SI-NEXT: s_mov_b32 s4, s7
-; SI-NEXT: s_mov_b32 s5, s8
-; SI-NEXT: s_mov_b32 s7, s9
-; SI-NEXT: s_mov_b32 s8, s10
-; SI-NEXT: s_mov_b32 s9, s11
-; SI-NEXT: s_mov_b32 s10, s12
-; SI-NEXT: s_mov_b32 s11, s13
-; SI-NEXT: s_mov_b32 s12, s14
-; SI-NEXT: s_mov_b32 s13, s15
-; SI-NEXT: s_mov_b32 s14, s40
-; SI-NEXT: s_mov_b32 s15, s41
-; SI-NEXT: s_mov_b32 s40, s42
-; SI-NEXT: s_mov_b32 s41, s43
-; SI-NEXT: s_mov_b32 s42, s44
-; SI-NEXT: s_mov_b32 s43, s45
-; SI-NEXT: s_mov_b32 s44, s46
-; SI-NEXT: s_mov_b32 s45, s47
-; SI-NEXT: s_mov_b32 s46, s56
-; SI-NEXT: s_mov_b32 s47, s57
-; SI-NEXT: s_mov_b32 s56, s58
-; SI-NEXT: s_mov_b32 s57, s59
-; SI-NEXT: s_mov_b32 s58, s60
-; SI-NEXT: v_readlane_b32 s59, v34, 1
-; SI-NEXT: v_readlane_b32 s60, v34, 2
-; SI-NEXT: v_readlane_b32 s61, v34, 3
-; SI-NEXT: v_readlane_b32 s62, v34, 4
-; SI-NEXT: v_readlane_b32 s63, v34, 5
-; SI-NEXT: v_readlane_b32 s72, v34, 6
-; SI-NEXT: v_readlane_b32 s73, v34, 7
-; SI-NEXT: v_readlane_b32 s74, v34, 8
-; SI-NEXT: v_readlane_b32 s6, v34, 9
-; SI-NEXT: v_readlane_b32 s75, v34, 10
-; SI-NEXT: v_readlane_b32 s76, v34, 11
-; SI-NEXT: v_readlane_b32 s79, v34, 12
-; SI-NEXT: v_readlane_b32 s88, v34, 13
-; SI-NEXT: v_readlane_b32 s89, v34, 14
-; SI-NEXT: v_readlane_b32 s90, v34, 15
-; SI-NEXT: v_readlane_b32 s91, v34, 16
-; SI-NEXT: v_readlane_b32 s92, v34, 17
-; SI-NEXT: v_readlane_b32 s93, v34, 18
-; SI-NEXT: v_readlane_b32 s94, v34, 19
-; SI-NEXT: v_readlane_b32 s95, v34, 20
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s63, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s63, 1
; SI-NEXT: s_cbranch_scc1 .LBB17_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_readlane_b32 s4, v34, 0
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_add_i32 s10, s10, 3
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_add_i32 s8, s8, 3
+; SI-NEXT: s_add_i32 s7, s7, 3
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: s_add_i32 s5, s5, 3
+; SI-NEXT: s_add_i32 s4, s4, 3
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_add_i32 s64, s64, 3
-; SI-NEXT: s_add_i32 s55, s55, 3
-; SI-NEXT: s_add_i32 s54, s54, 3
-; SI-NEXT: s_add_i32 s53, s53, 3
-; SI-NEXT: s_add_i32 s52, s52, 3
-; SI-NEXT: s_add_i32 s51, s51, 3
-; SI-NEXT: s_add_i32 s50, s50, 3
-; SI-NEXT: s_add_i32 s49, s49, 3
-; SI-NEXT: s_add_i32 s48, s48, 3
-; SI-NEXT: s_add_i32 s39, s39, 3
-; SI-NEXT: s_add_i32 s38, s38, 3
-; SI-NEXT: s_add_i32 s37, s37, 3
-; SI-NEXT: s_add_i32 s36, s36, 3
-; SI-NEXT: s_add_i32 s35, s35, 3
-; SI-NEXT: s_add_i32 s34, s34, 3
-; SI-NEXT: s_add_i32 s31, s31, 3
-; SI-NEXT: s_add_i32 s30, s30, 3
-; SI-NEXT: s_add_i32 s4, s4, 3
-; SI-NEXT: s_and_b32 s5, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s8, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s30, 16
-; SI-NEXT: s_and_b32 s10, s31, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s31, 16
-; SI-NEXT: s_and_b32 s12, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s34, 16
-; SI-NEXT: s_and_b32 s14, s35, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s35, 16
-; SI-NEXT: s_and_b32 s40, s36, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s36, 16
-; SI-NEXT: s_and_b32 s42, s37, 0xffff0000
-; SI-NEXT: s_lshl_b32 s41, s37, 16
-; SI-NEXT: s_and_b32 s44, s38, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s38, 16
-; SI-NEXT: s_and_b32 s46, s39, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s39, 16
-; SI-NEXT: s_and_b32 s56, s48, 0xffff0000
-; SI-NEXT: s_lshl_b32 s47, s48, 16
-; SI-NEXT: s_and_b32 s58, s49, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s49, 16
-; SI-NEXT: s_and_b32 s60, s50, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s50, 16
-; SI-NEXT: s_and_b32 s62, s51, 0xffff0000
-; SI-NEXT: s_lshl_b32 s61, s51, 16
-; SI-NEXT: s_and_b32 s72, s52, 0xffff0000
-; SI-NEXT: s_lshl_b32 s63, s52, 16
-; SI-NEXT: s_and_b32 s74, s53, 0xffff0000
-; SI-NEXT: s_lshl_b32 s73, s53, 16
-; SI-NEXT: s_and_b32 s78, s54, 0xffff0000
-; SI-NEXT: s_lshl_b32 s77, s54, 16
-; SI-NEXT: s_and_b32 s75, s55, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s55, 16
-; SI-NEXT: s_and_b32 s79, s64, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s64, 16
-; SI-NEXT: s_and_b32 s89, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s29, 16
-; SI-NEXT: s_and_b32 s91, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s28, 16
-; SI-NEXT: s_and_b32 s93, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s27, 16
-; SI-NEXT: s_and_b32 s95, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s26, 16
-; SI-NEXT: s_and_b32 vcc_hi, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, s25, 16
-; SI-NEXT: s_and_b32 s66, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s65, s24, 16
-; SI-NEXT: s_and_b32 s68, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s23, 16
-; SI-NEXT: s_and_b32 s70, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s69, s22, 16
-; SI-NEXT: s_and_b32 s80, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s21, 16
-; SI-NEXT: s_and_b32 s82, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s20, 16
-; SI-NEXT: s_and_b32 s84, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, s19, 16
-; SI-NEXT: s_and_b32 s86, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s18, 16
-; SI-NEXT: s_and_b32 s96, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s17, 16
-; SI-NEXT: s_and_b32 s98, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s97, s16, 16
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s45, s45, 3
+; SI-NEXT: s_add_i32 s44, s44, 3
+; SI-NEXT: s_add_i32 s43, s43, 3
+; SI-NEXT: s_add_i32 s42, s42, 3
+; SI-NEXT: s_add_i32 s41, s41, 3
+; SI-NEXT: s_add_i32 s40, s40, 3
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: s_add_i32 s14, s14, 3
+; SI-NEXT: s_add_i32 s13, s13, 3
+; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[56:57], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[28:29], 16
+; SI-NEXT: s_lshr_b32 s47, s5, 16
+; SI-NEXT: s_lshr_b32 s38, s7, 16
+; SI-NEXT: s_lshr_b32 s39, s9, 16
+; SI-NEXT: s_lshr_b32 s48, s11, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s50, s15, 16
+; SI-NEXT: s_lshr_b32 s51, s41, 16
+; SI-NEXT: s_lshr_b32 s52, s43, 16
+; SI-NEXT: s_lshr_b32 s57, s45, 16
+; SI-NEXT: s_lshr_b32 s53, s29, 16
+; SI-NEXT: s_lshr_b32 s54, s27, 16
+; SI-NEXT: s_lshr_b32 s55, s25, 16
+; SI-NEXT: s_lshr_b32 s64, s23, 16
+; SI-NEXT: s_lshr_b32 s65, s21, 16
+; SI-NEXT: s_lshr_b32 s59, s19, 16
+; SI-NEXT: s_lshr_b32 s61, s17, 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
; SI-NEXT: .LBB17_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s98
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s97
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s96
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s87
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s86
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s85
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s84
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s83
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s82
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s81
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s80
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s71
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s70
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s69
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s68
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s67
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s66
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s65
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, vcc_hi
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, vcc_lo
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s95
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s94
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s93
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s92
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s91
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s90
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s89
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s88
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s79
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s76
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s78
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s77
-; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s74
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s73
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s72
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s63
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s62
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s61
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s60
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s59
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s58
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s57
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s56
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s47
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s46
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s45
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s44
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s43
-; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s42
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s41
-; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s40
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s15
-; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s14
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s13
-; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s12
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s11
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s10
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s5
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
-; SI-NEXT: v_readlane_b32 s30, v33, 33
-; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
-; SI-NEXT: v_readlane_b32 s31, v33, 34
-; SI-NEXT: v_readlane_b32 s98, v33, 32
-; SI-NEXT: v_readlane_b32 s97, v33, 31
-; SI-NEXT: v_readlane_b32 s96, v33, 30
-; SI-NEXT: v_readlane_b32 s87, v33, 29
-; SI-NEXT: v_readlane_b32 s86, v33, 28
-; SI-NEXT: v_readlane_b32 s85, v33, 27
-; SI-NEXT: v_readlane_b32 s84, v33, 26
-; SI-NEXT: v_readlane_b32 s83, v33, 25
-; SI-NEXT: v_readlane_b32 s82, v33, 24
-; SI-NEXT: v_readlane_b32 s81, v33, 23
-; SI-NEXT: v_readlane_b32 s80, v33, 22
-; SI-NEXT: v_readlane_b32 s71, v33, 21
-; SI-NEXT: v_readlane_b32 s70, v33, 20
-; SI-NEXT: v_readlane_b32 s69, v33, 19
-; SI-NEXT: v_readlane_b32 s68, v33, 18
-; SI-NEXT: v_readlane_b32 s67, v33, 17
-; SI-NEXT: v_readlane_b32 s66, v33, 16
-; SI-NEXT: v_readlane_b32 s65, v33, 15
-; SI-NEXT: v_readlane_b32 s64, v33, 14
-; SI-NEXT: v_readlane_b32 s55, v33, 13
-; SI-NEXT: v_readlane_b32 s54, v33, 12
-; SI-NEXT: v_readlane_b32 s53, v33, 11
-; SI-NEXT: v_readlane_b32 s52, v33, 10
-; SI-NEXT: v_readlane_b32 s51, v33, 9
-; SI-NEXT: v_readlane_b32 s50, v33, 8
-; SI-NEXT: v_readlane_b32 s49, v33, 7
-; SI-NEXT: v_readlane_b32 s48, v33, 6
-; SI-NEXT: v_readlane_b32 s39, v33, 5
-; SI-NEXT: v_readlane_b32 s38, v33, 4
-; SI-NEXT: v_readlane_b32 s37, v33, 3
-; SI-NEXT: v_readlane_b32 s36, v33, 2
-; SI-NEXT: v_readlane_b32 s35, v33, 1
-; SI-NEXT: v_readlane_b32 s34, v33, 0
+; SI-NEXT: s_and_b32 s45, s45, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s57, 16
+; SI-NEXT: s_and_b32 s19, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s59, 16
+; SI-NEXT: s_or_b32 s45, s45, s57
+; SI-NEXT: s_and_b32 s42, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s76, 16
+; SI-NEXT: s_or_b32 s19, s19, s59
+; SI-NEXT: s_lshl_b32 s59, vcc_lo, 16
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s42, s42, s57
+; SI-NEXT: s_and_b32 s43, s43, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s52, 16
+; SI-NEXT: s_or_b32 s20, s20, s59
+; SI-NEXT: s_and_b32 s21, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s65, 16
+; SI-NEXT: s_or_b32 s43, s43, s57
+; SI-NEXT: s_and_b32 s40, s40, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s74, 16
+; SI-NEXT: s_or_b32 s21, s21, s59
+; SI-NEXT: s_lshl_b32 s59, s94, 16
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s40, s40, s57
+; SI-NEXT: s_and_b32 s41, s41, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s51, 16
+; SI-NEXT: s_or_b32 s22, s22, s59
+; SI-NEXT: s_and_b32 s23, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s64, 16
+; SI-NEXT: s_or_b32 s41, s41, s57
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s72, 16
+; SI-NEXT: s_or_b32 s23, s23, s59
+; SI-NEXT: s_lshl_b32 s59, s92, 16
+; SI-NEXT: s_and_b32 s24, s24, 0xffff
+; SI-NEXT: s_or_b32 s14, s14, s57
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s50, 16
+; SI-NEXT: s_or_b32 s24, s24, s59
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s55, 16
+; SI-NEXT: s_or_b32 s15, s15, s57
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s62, 16
+; SI-NEXT: s_or_b32 s25, s25, s59
+; SI-NEXT: s_and_b32 s26, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s90, 16
+; SI-NEXT: s_or_b32 s12, s12, s57
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s49, 16
+; SI-NEXT: s_or_b32 s26, s26, s59
+; SI-NEXT: s_and_b32 s27, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s54, 16
+; SI-NEXT: s_or_b32 s13, s13, s57
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s58, 16
+; SI-NEXT: s_or_b32 s27, s27, s59
+; SI-NEXT: s_and_b32 s28, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s88, 16
+; SI-NEXT: s_or_b32 s10, s10, s57
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s48, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s61, s61, 16
+; SI-NEXT: s_or_b32 s28, s28, s59
+; SI-NEXT: s_and_b32 s29, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s53, 16
+; SI-NEXT: s_or_b32 s11, s11, s57
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s60, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_lshl_b32 s63, s34, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s61
+; SI-NEXT: s_lshl_b32 s61, s30, 16
+; SI-NEXT: s_and_b32 s18, s18, 0xffff
+; SI-NEXT: s_or_b32 s29, s29, s59
+; SI-NEXT: s_and_b32 s44, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s59, s78, 16
+; SI-NEXT: s_or_b32 s8, s8, s57
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s39, 16
+; SI-NEXT: s_or_b32 s6, s6, s56
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s38, 16
+; SI-NEXT: s_or_b32 s4, s4, s46
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s47, 16
+; SI-NEXT: s_or_b32 s16, s16, s63
+; SI-NEXT: s_or_b32 s18, s18, s61
+; SI-NEXT: s_or_b32 s44, s44, s59
+; SI-NEXT: s_or_b32 s9, s9, s57
+; SI-NEXT: s_or_b32 s7, s7, s56
+; SI-NEXT: s_or_b32 s5, s5, s46
+; SI-NEXT: v_readlane_b32 s30, v32, 16
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v14, s44
+; SI-NEXT: v_mov_b32_e32 v15, s45
+; SI-NEXT: v_mov_b32_e32 v16, s42
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v18, s40
+; SI-NEXT: v_mov_b32_e32 v19, s41
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v21, s15
+; SI-NEXT: v_mov_b32_e32 v22, s12
+; SI-NEXT: v_mov_b32_e32 v23, s13
+; SI-NEXT: v_mov_b32_e32 v24, s10
+; SI-NEXT: v_mov_b32_e32 v25, s11
+; SI-NEXT: v_mov_b32_e32 v26, s8
+; SI-NEXT: v_mov_b32_e32 v27, s9
+; SI-NEXT: v_mov_b32_e32 v28, s6
+; SI-NEXT: v_mov_b32_e32 v29, s7
+; SI-NEXT: v_mov_b32_e32 v30, s4
+; SI-NEXT: v_mov_b32_e32 v31, s5
+; SI-NEXT: v_readlane_b32 s31, v32, 17
+; SI-NEXT: v_readlane_b32 s65, v32, 15
+; SI-NEXT: v_readlane_b32 s64, v32, 14
+; SI-NEXT: v_readlane_b32 s55, v32, 13
+; SI-NEXT: v_readlane_b32 s54, v32, 12
+; SI-NEXT: v_readlane_b32 s53, v32, 11
+; SI-NEXT: v_readlane_b32 s52, v32, 10
+; SI-NEXT: v_readlane_b32 s51, v32, 9
+; SI-NEXT: v_readlane_b32 s50, v32, 8
+; SI-NEXT: v_readlane_b32 s49, v32, 7
+; SI-NEXT: v_readlane_b32 s48, v32, 6
+; SI-NEXT: v_readlane_b32 s39, v32, 5
+; SI-NEXT: v_readlane_b32 s38, v32, 4
+; SI-NEXT: v_readlane_b32 s37, v32, 3
+; SI-NEXT: v_readlane_b32 s36, v32, 2
+; SI-NEXT: v_readlane_b32 s35, v32, 1
+; SI-NEXT: v_readlane_b32 s34, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -23633,265 +23063,308 @@ define <32 x i32> @bitcast_v64bf16_to_v32i32(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v62, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v63
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32
+; SI-NEXT: v_mov_b32_e32 v44, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; SI-NEXT: v_mov_b32_e32 v45, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v19
+; SI-NEXT: v_mov_b32_e32 v46, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v20
+; SI-NEXT: v_mov_b32_e32 v47, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
+; SI-NEXT: v_mov_b32_e32 v56, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_mov_b32_e32 v57, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v23
+; SI-NEXT: v_mov_b32_e32 v58, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v24
+; SI-NEXT: v_mov_b32_e32 v59, v3
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v25
+; SI-NEXT: v_mov_b32_e32 v60, v2
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v26
+; SI-NEXT: v_mov_b32_e32 v61, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v27
+; SI-NEXT: v_mov_b32_e32 v42, v12
+; SI-NEXT: v_mov_b32_e32 v43, v11
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v17
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v62
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v42
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v43
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v45
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v46
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v56
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v57
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v58
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v59
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v60
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v61
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v23
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v11
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v60
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v53
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v37
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v38
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v38
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v7
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v63
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v51
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v35
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v34
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB18_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v3, v3, v32
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v62
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v60
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v58
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v56
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v47
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v46
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v45
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v44
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v43
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: v_or_b32_e32 v0, v0, v41
+; SI-NEXT: v_or_b32_e32 v1, v1, v48
+; SI-NEXT: v_or_b32_e32 v2, v2, v40
+; SI-NEXT: v_or_b32_e32 v4, v4, v55
+; SI-NEXT: v_or_b32_e32 v5, v5, v39
+; SI-NEXT: v_or_b32_e32 v6, v6, v54
+; SI-NEXT: v_or_b32_e32 v7, v7, v35
+; SI-NEXT: v_or_b32_e32 v8, v8, v53
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v34
+; SI-NEXT: v_or_b32_e32 v12, v12, v51
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v18, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v13, v13, v37
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v15, v15, v33
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v16, v16, v49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v17, v17, v36
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v63
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
@@ -23925,393 +23398,238 @@ define <32 x i32> @bitcast_v64bf16_to_v32i32(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v43
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v34
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_alignbit_b32 v19, v19, v59, 16
-; SI-NEXT: v_alignbit_b32 v20, v20, v57, 16
-; SI-NEXT: v_alignbit_b32 v21, v21, v46, 16
-; SI-NEXT: v_alignbit_b32 v22, v22, v44, 16
-; SI-NEXT: v_alignbit_b32 v23, v23, v42, 16
-; SI-NEXT: v_alignbit_b32 v24, v24, v40, 16
-; SI-NEXT: v_alignbit_b32 v25, v25, v54, 16
-; SI-NEXT: v_alignbit_b32 v26, v26, v52, 16
-; SI-NEXT: v_alignbit_b32 v27, v27, v50, 16
-; SI-NEXT: v_alignbit_b32 v28, v28, v48, 16
-; SI-NEXT: v_alignbit_b32 v29, v29, v38, 16
-; SI-NEXT: v_alignbit_b32 v30, v30, v35, 16
-; SI-NEXT: v_alignbit_b32 v31, v31, v33, 16
; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: .LBB18_2: ; %Flow
+; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB18_4
+; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v41
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v61
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v48
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_alignbit_b32 v0, v0, v63, 16
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v60
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v59
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v32
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v58
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v55
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v57
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v39
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v54
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v47
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v35
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v46
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v45
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v44
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v43
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v34
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v42
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v51
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v37
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v50
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v62
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v60
-; SI-NEXT: v_alignbit_b32 v18, v18, v61, 16
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: .LBB18_2: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB18_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v49
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v36
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v60
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v58
+; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v47
+; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v45
+; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v43
+; SI-NEXT: s_waitcnt vmcnt(10)
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v41
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v55
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v53
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v51
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v49
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v39
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v36
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v34
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v32
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_alignbit_b32 v15, v16, v15, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_alignbit_b32 v16, v17, v16, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_alignbit_b32 v17, v18, v17, 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v61
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_alignbit_b32 v18, v19, v18, 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v59
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_alignbit_b32 v19, v20, v19, 16
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v57
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_alignbit_b32 v20, v21, v20, 16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v46
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_alignbit_b32 v21, v22, v21, 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v44
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_alignbit_b32 v22, v23, v22, 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v42
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_alignbit_b32 v23, v24, v23, 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_alignbit_b32 v24, v25, v24, 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v54
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_alignbit_b32 v25, v26, v25, 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v52
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_alignbit_b32 v26, v27, v26, 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v50
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_alignbit_b32 v27, v28, v27, 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v48
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_alignbit_b32 v28, v29, v28, 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v38
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_alignbit_b32 v29, v30, v29, 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v35
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_alignbit_b32 v30, v31, v30, 16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v33
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v63
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_alignbit_b32 v31, v32, v31, 16
; SI-NEXT: .LBB18_4: ; %end
@@ -26667,768 +25985,486 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-LABEL: bitcast_v64bf16_to_v32i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_writelane_b32 v63, s34, 0
-; SI-NEXT: v_writelane_b32 v63, s35, 1
-; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s30, 3
-; SI-NEXT: v_writelane_b32 v63, s31, 4
-; SI-NEXT: v_readfirstlane_b32 s78, v7
-; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s72, v10
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
-; SI-NEXT: v_readfirstlane_b32 s62, v11
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
-; SI-NEXT: v_readfirstlane_b32 s60, v12
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
-; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
-; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s94
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s92, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s92
-; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s90
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v18
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s42
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s78
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v36, s34, 0
+; SI-NEXT: v_writelane_b32 v36, s35, 1
+; SI-NEXT: v_writelane_b32 v36, s36, 2
+; SI-NEXT: v_writelane_b32 v36, s37, 3
+; SI-NEXT: v_writelane_b32 v36, s38, 4
+; SI-NEXT: v_writelane_b32 v36, s39, 5
+; SI-NEXT: v_writelane_b32 v36, s48, 6
+; SI-NEXT: v_writelane_b32 v36, s49, 7
+; SI-NEXT: v_writelane_b32 v36, s50, 8
+; SI-NEXT: v_writelane_b32 v36, s51, 9
+; SI-NEXT: v_writelane_b32 v36, s52, 10
+; SI-NEXT: v_writelane_b32 v36, s53, 11
+; SI-NEXT: v_writelane_b32 v36, s54, 12
+; SI-NEXT: v_writelane_b32 v36, s55, 13
+; SI-NEXT: v_writelane_b32 v36, s64, 14
+; SI-NEXT: v_writelane_b32 v36, s65, 15
+; SI-NEXT: v_writelane_b32 v36, s66, 16
+; SI-NEXT: v_writelane_b32 v36, s67, 17
+; SI-NEXT: v_writelane_b32 v36, s68, 18
+; SI-NEXT: v_writelane_b32 v36, s69, 19
+; SI-NEXT: v_writelane_b32 v36, s70, 20
+; SI-NEXT: v_writelane_b32 v36, s71, 21
+; SI-NEXT: v_writelane_b32 v36, s80, 22
+; SI-NEXT: v_writelane_b32 v36, s81, 23
+; SI-NEXT: v_writelane_b32 v36, s82, 24
+; SI-NEXT: v_writelane_b32 v36, s83, 25
+; SI-NEXT: v_writelane_b32 v36, s84, 26
+; SI-NEXT: v_writelane_b32 v36, s85, 27
+; SI-NEXT: v_writelane_b32 v36, s86, 28
+; SI-NEXT: v_writelane_b32 v36, s87, 29
+; SI-NEXT: v_writelane_b32 v36, s96, 30
+; SI-NEXT: v_writelane_b32 v36, s97, 31
+; SI-NEXT: v_writelane_b32 v36, s98, 32
+; SI-NEXT: v_writelane_b32 v36, s99, 33
+; SI-NEXT: v_writelane_b32 v36, s30, 34
+; SI-NEXT: v_writelane_b32 v36, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
+; SI-NEXT: v_readfirstlane_b32 s11, v15
+; SI-NEXT: v_readfirstlane_b32 s13, v14
+; SI-NEXT: v_readfirstlane_b32 s15, v13
+; SI-NEXT: v_readfirstlane_b32 s73, v12
+; SI-NEXT: v_readfirstlane_b32 s75, v11
+; SI-NEXT: v_readfirstlane_b32 s77, v10
+; SI-NEXT: v_readfirstlane_b32 s79, v9
+; SI-NEXT: v_readfirstlane_b32 s89, v8
+; SI-NEXT: v_readfirstlane_b32 s91, v7
+; SI-NEXT: v_readfirstlane_b32 s93, v6
+; SI-NEXT: v_readfirstlane_b32 s95, v5
+; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s34, v3
+; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: v_readfirstlane_b32 s80, v1
+; SI-NEXT: v_readfirstlane_b32 s83, v0
+; SI-NEXT: s_lshr_b32 vcc_lo, s29, 16
+; SI-NEXT: s_lshr_b32 s30, s28, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s70, s26, 16
+; SI-NEXT: s_lshr_b32 s81, s25, 16
+; SI-NEXT: s_lshr_b32 s84, s24, 16
+; SI-NEXT: s_lshr_b32 s85, s23, 16
+; SI-NEXT: s_lshr_b32 s86, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s87, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s66, s9, 16
+; SI-NEXT: s_lshr_b32 s10, s11, 16
+; SI-NEXT: s_lshr_b32 s12, s13, 16
+; SI-NEXT: s_lshr_b32 s14, s15, 16
+; SI-NEXT: s_lshr_b32 s72, s73, 16
+; SI-NEXT: s_lshr_b32 s74, s75, 16
+; SI-NEXT: s_lshr_b32 s76, s77, 16
+; SI-NEXT: s_lshr_b32 s78, s79, 16
+; SI-NEXT: s_lshr_b32 s88, s89, 16
+; SI-NEXT: s_lshr_b32 s90, s91, 16
+; SI-NEXT: s_lshr_b32 s92, s93, 16
+; SI-NEXT: s_lshr_b32 s94, s95, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s4, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s69, 16
+; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 s82, s83, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: ; implicit-def: $vgpr37 : SGPR spill to VGPR lane
+; SI-NEXT: s_cmp_lg_u32 s5, 0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v37, s67, 0
+; SI-NEXT: v_writelane_b32 v37, s66, 1
; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v37
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v48
-; SI-NEXT: v_mov_b32_e32 v62, v35
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v18
-; SI-NEXT: v_mov_b32_e32 v29, v54
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v30, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s36, s5, s6
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s99, 16
+; SI-NEXT: s_or_b32 s38, s5, s6
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s98, 16
+; SI-NEXT: s_and_b32 s40, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s41, s87, 16
+; SI-NEXT: s_or_b32 s39, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s97, 16
+; SI-NEXT: s_or_b32 s37, s40, s41
+; SI-NEXT: s_or_b32 s40, s5, s6
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s96, 16
+; SI-NEXT: s_or_b32 s41, s5, s6
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s86, 16
+; SI-NEXT: s_or_b32 s42, s5, s6
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s85, 16
+; SI-NEXT: s_or_b32 s43, s5, s6
+; SI-NEXT: s_and_b32 s5, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s84, 16
+; SI-NEXT: s_or_b32 s44, s5, s6
+; SI-NEXT: s_and_b32 s5, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s81, 16
+; SI-NEXT: s_or_b32 s45, s5, s6
+; SI-NEXT: s_and_b32 s5, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s70, 16
+; SI-NEXT: s_or_b32 s46, s5, s6
+; SI-NEXT: s_and_b32 s5, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s35, 16
+; SI-NEXT: s_or_b32 s47, s5, s6
+; SI-NEXT: s_and_b32 s5, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s30, 16
+; SI-NEXT: s_or_b32 s48, s5, s6
+; SI-NEXT: s_and_b32 s5, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s6, vcc_lo, 16
+; SI-NEXT: s_or_b32 s49, s5, s6
+; SI-NEXT: s_and_b32 s5, s83, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s82, 16
+; SI-NEXT: s_or_b32 s50, s5, s6
+; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s71, 16
+; SI-NEXT: s_or_b32 s51, s5, s6
+; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s68, 16
+; SI-NEXT: s_or_b32 s52, s5, s6
+; SI-NEXT: s_and_b32 s5, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s31, 16
+; SI-NEXT: s_or_b32 s53, s5, s6
+; SI-NEXT: s_mov_b32 s6, s4
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_or_b32 s54, s4, s5
+; SI-NEXT: s_and_b32 s4, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_or_b32 s55, s4, s5
+; SI-NEXT: s_and_b32 s4, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_or_b32 s56, s4, s5
+; SI-NEXT: s_and_b32 s4, s91, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_or_b32 s57, s4, s5
+; SI-NEXT: s_and_b32 s4, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_or_b32 s58, s4, s5
+; SI-NEXT: s_and_b32 s4, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: s_or_b32 s59, s4, s5
+; SI-NEXT: s_and_b32 s4, s77, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_or_b32 s60, s4, s5
+; SI-NEXT: s_and_b32 s4, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_or_b32 s61, s4, s5
+; SI-NEXT: s_and_b32 s4, s73, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_or_b32 s62, s4, s5
+; SI-NEXT: s_and_b32 s4, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s63, s4, s5
+; SI-NEXT: s_and_b32 s4, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s64, s4, s5
+; SI-NEXT: s_and_b32 s4, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s10, 16
+; SI-NEXT: s_or_b32 s65, s4, s5
+; SI-NEXT: s_and_b32 s4, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: s_or_b32 s66, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_or_b32 s67, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(2) expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[38:39], 16
-; SI-NEXT: v_mov_b32_e32 v39, v33
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v39
-; SI-NEXT: v_lshr_b64 v[3:4], v[32:33], 16
-; SI-NEXT: v_mov_b32_e32 v4, v41
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[41:42], 16
-; SI-NEXT: v_mov_b32_e32 v5, v26
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[26:27], 16
-; SI-NEXT: v_mov_b32_e32 v6, v25
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v7, v24
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v8
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[24:25], 16
-; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v9, v13
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v33, v61
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v33
-; SI-NEXT: v_lshr_b64 v[10:11], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v13, v60
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[59:60], 16
-; SI-NEXT: v_mov_b32_e32 v14, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v15
-; SI-NEXT: v_mov_b32_e32 v60, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[58:59], 16
-; SI-NEXT: v_mov_b32_e32 v15, v58
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v16
-; SI-NEXT: v_mov_b32_e32 v59, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[57:58], 16
-; SI-NEXT: v_mov_b32_e32 v16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v17
-; SI-NEXT: v_mov_b32_e32 v58, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v17, v56
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v19
-; SI-NEXT: v_mov_b32_e32 v57, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[55:56], 16
-; SI-NEXT: v_mov_b32_e32 v56, v17
-; SI-NEXT: v_lshr_b64 v[16:17], v[51:52], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v20
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v42, v49
-; SI-NEXT: v_mov_b32_e32 v49, v44
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v27
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v54
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v25
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v21
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v22
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v50
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v26
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v24, v53
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[53:54], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v54, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v31
-; SI-NEXT: v_mov_b32_e32 v31, v50
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
-; SI-NEXT: v_lshr_b64 v[25:26], v[40:41], 16
-; SI-NEXT: v_mov_b32_e32 v26, v43
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[26:27], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v27, v28
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v37
-; SI-NEXT: v_mov_b32_e32 v37, v48
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v35
-; SI-NEXT: v_mov_b32_e32 v35, v62
-; SI-NEXT: v_lshr_b64 v[61:62], v[47:48], 16
-; SI-NEXT: v_mov_b32_e32 v44, v49
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[45:46], 16
-; SI-NEXT: v_mov_b32_e32 v46, v30
-; SI-NEXT: v_lshr_b64 v[29:30], v[49:50], 16
-; SI-NEXT: v_mov_b32_e32 v50, v31
-; SI-NEXT: v_lshr_b64 v[30:31], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v61, v33
-; SI-NEXT: v_mov_b32_e32 v33, v39
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v49, v42
; SI-NEXT: s_branch .LBB19_3
; SI-NEXT: .LBB19_2:
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b32 s6, s4
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
; SI-NEXT: .LBB19_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB19_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v31, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v35
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s87, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s99, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s98, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s97, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s96, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s86, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s85, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s84, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s81, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s70, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s35, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_lo, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: s_lshl_b32 s4, s83, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s82, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_lshl_b32 s4, s80, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s71, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: s_lshl_b32 s4, s69, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s68, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: s_lshl_b32 s4, s34, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s31, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: s_lshl_b32 s4, s95, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s94, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: s_lshl_b32 s4, s93, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s92, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_lshl_b32 s4, s91, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s90, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: s_lshl_b32 s4, s89, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s88, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: s_lshl_b32 s4, s79, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s78, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s77, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s76, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s74, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s72, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_add_f32_e32 v32, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 1
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v31, v32
-; SI-NEXT: .LBB19_5: ; %end
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s30, v63, 3
-; SI-NEXT: v_readlane_b32 s31, v63, 4
-; SI-NEXT: v_readlane_b32 s36, v63, 2
-; SI-NEXT: v_readlane_b32 s35, v63, 1
-; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 0
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v31, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_lshr_b64 v[34:35], v[30:31], 16
+; SI-NEXT: v_lshr_b64 v[30:31], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v31, v34
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: v_mov_b32_e32 v16, s52
+; SI-NEXT: v_mov_b32_e32 v17, s53
+; SI-NEXT: v_mov_b32_e32 v18, s54
+; SI-NEXT: v_mov_b32_e32 v19, s55
+; SI-NEXT: v_mov_b32_e32 v20, s56
+; SI-NEXT: v_mov_b32_e32 v21, s57
+; SI-NEXT: v_mov_b32_e32 v22, s58
+; SI-NEXT: v_mov_b32_e32 v23, s59
+; SI-NEXT: v_mov_b32_e32 v24, s60
+; SI-NEXT: v_mov_b32_e32 v25, s61
+; SI-NEXT: v_mov_b32_e32 v26, s62
+; SI-NEXT: v_mov_b32_e32 v27, s63
+; SI-NEXT: v_mov_b32_e32 v28, s64
+; SI-NEXT: v_mov_b32_e32 v29, s65
+; SI-NEXT: v_mov_b32_e32 v30, s66
+; SI-NEXT: v_mov_b32_e32 v31, s67
+; SI-NEXT: .LBB19_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v36, 34
+; SI-NEXT: v_readlane_b32 s31, v36, 35
+; SI-NEXT: v_readlane_b32 s99, v36, 33
+; SI-NEXT: v_readlane_b32 s98, v36, 32
+; SI-NEXT: v_readlane_b32 s97, v36, 31
+; SI-NEXT: v_readlane_b32 s96, v36, 30
+; SI-NEXT: v_readlane_b32 s87, v36, 29
+; SI-NEXT: v_readlane_b32 s86, v36, 28
+; SI-NEXT: v_readlane_b32 s85, v36, 27
+; SI-NEXT: v_readlane_b32 s84, v36, 26
+; SI-NEXT: v_readlane_b32 s83, v36, 25
+; SI-NEXT: v_readlane_b32 s82, v36, 24
+; SI-NEXT: v_readlane_b32 s81, v36, 23
+; SI-NEXT: v_readlane_b32 s80, v36, 22
+; SI-NEXT: v_readlane_b32 s71, v36, 21
+; SI-NEXT: v_readlane_b32 s70, v36, 20
+; SI-NEXT: v_readlane_b32 s69, v36, 19
+; SI-NEXT: v_readlane_b32 s68, v36, 18
+; SI-NEXT: v_readlane_b32 s67, v36, 17
+; SI-NEXT: v_readlane_b32 s66, v36, 16
+; SI-NEXT: v_readlane_b32 s65, v36, 15
+; SI-NEXT: v_readlane_b32 s64, v36, 14
+; SI-NEXT: v_readlane_b32 s55, v36, 13
+; SI-NEXT: v_readlane_b32 s54, v36, 12
+; SI-NEXT: v_readlane_b32 s53, v36, 11
+; SI-NEXT: v_readlane_b32 s52, v36, 10
+; SI-NEXT: v_readlane_b32 s51, v36, 9
+; SI-NEXT: v_readlane_b32 s50, v36, 8
+; SI-NEXT: v_readlane_b32 s49, v36, 7
+; SI-NEXT: v_readlane_b32 s48, v36, 6
+; SI-NEXT: v_readlane_b32 s39, v36, 5
+; SI-NEXT: v_readlane_b32 s38, v36, 4
+; SI-NEXT: v_readlane_b32 s37, v36, 3
+; SI-NEXT: v_readlane_b32 s36, v36, 2
+; SI-NEXT: v_readlane_b32 s35, v36, 1
+; SI-NEXT: v_readlane_b32 s34, v36, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64bf16_to_v32i32_scalar:
@@ -58072,596 +57108,223 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr62
; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr60
; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr59
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr58
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB40_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v62
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v53, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v55, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v42, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v44, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v47, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v62
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v13
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v12
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v11
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v10
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v9
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v8
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v7
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v2
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v1
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB40_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB40_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_f32_e32 v31, 1.0, v62
-; SI-NEXT: v_add_f32_e32 v30, 1.0, v30
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_add_f32_e32 v29, 1.0, v29
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_add_f32_e32 v28, 1.0, v28
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_add_f32_e32 v27, 1.0, v27
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_add_f32_e32 v26, 1.0, v26
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_add_f32_e32 v25, 1.0, v25
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_add_f32_e32 v24, 1.0, v24
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_add_f32_e32 v23, 1.0, v23
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_add_f32_e32 v22, 1.0, v22
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_add_f32_e32 v21, 1.0, v21
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_add_f32_e32 v20, 1.0, v20
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_add_f32_e32 v19, 1.0, v19
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_f32_e32 v18, 1.0, v18
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_add_f32_e32 v17, 1.0, v17
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
+; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_add_f32_e32 v4, 1.0, v4
+; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
; SI-NEXT: v_add_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_add_f32_e32 v6, 1.0, v6
+; SI-NEXT: v_add_f32_e32 v4, 1.0, v4
; SI-NEXT: v_add_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_add_f32_e32 v8, 1.0, v8
+; SI-NEXT: v_add_f32_e32 v6, 1.0, v6
; SI-NEXT: v_add_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_add_f32_e32 v10, 1.0, v10
+; SI-NEXT: v_add_f32_e32 v8, 1.0, v8
; SI-NEXT: v_add_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_add_f32_e32 v12, 1.0, v12
+; SI-NEXT: v_add_f32_e32 v10, 1.0, v10
; SI-NEXT: v_add_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_add_f32_e32 v14, 1.0, v14
+; SI-NEXT: v_add_f32_e32 v12, 1.0, v12
; SI-NEXT: v_add_f32_e32 v15, 1.0, v15
+; SI-NEXT: v_add_f32_e32 v14, 1.0, v14
+; SI-NEXT: v_add_f32_e32 v17, 1.0, v17
; SI-NEXT: v_add_f32_e32 v16, 1.0, v16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v19, 1.0, v19
+; SI-NEXT: v_add_f32_e32 v18, 1.0, v18
+; SI-NEXT: v_add_f32_e32 v21, 1.0, v21
+; SI-NEXT: v_add_f32_e32 v20, 1.0, v20
+; SI-NEXT: v_add_f32_e32 v23, 1.0, v23
+; SI-NEXT: v_add_f32_e32 v22, 1.0, v22
+; SI-NEXT: v_add_f32_e32 v25, 1.0, v25
+; SI-NEXT: v_add_f32_e32 v24, 1.0, v24
+; SI-NEXT: v_add_f32_e32 v27, 1.0, v27
+; SI-NEXT: v_add_f32_e32 v26, 1.0, v26
+; SI-NEXT: v_add_f32_e32 v29, 1.0, v29
+; SI-NEXT: v_add_f32_e32 v28, 1.0, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v31, 1.0, v31
+; SI-NEXT: v_add_f32_e32 v30, 1.0, v30
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v53, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v55, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v42, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v44, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v47, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v13
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v12
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v11
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v10
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v9
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v8
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v7
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v2
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v1
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB40_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v60
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v58
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v56
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v46
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v44
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v42
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v40
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; SI-NEXT: v_or_b32_e32 v0, v0, v47
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v63
+; SI-NEXT: v_or_b32_e32 v2, v2, v44
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v62
+; SI-NEXT: v_or_b32_e32 v4, v4, v42
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v61
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_or_b32_e32 v1, v1, v47
+; SI-NEXT: v_or_b32_e32 v3, v3, v44
+; SI-NEXT: v_or_b32_e32 v5, v5, v42
+; SI-NEXT: v_or_b32_e32 v6, v6, v55
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v60
+; SI-NEXT: v_or_b32_e32 v8, v8, v53
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v59
+; SI-NEXT: v_or_b32_e32 v10, v10, v50
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v58
+; SI-NEXT: v_or_b32_e32 v12, v12, v49
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v57
+; SI-NEXT: v_or_b32_e32 v14, v14, v48
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v56
+; SI-NEXT: v_or_b32_e32 v16, v16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v46
+; SI-NEXT: v_or_b32_e32 v18, v18, v38
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v45
+; SI-NEXT: v_or_b32_e32 v20, v20, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v43
+; SI-NEXT: v_or_b32_e32 v22, v22, v36
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v41
+; SI-NEXT: v_or_b32_e32 v24, v24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v40
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v63
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@@ -58678,8 +57341,45 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v26, v26, v34
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v54
+; SI-NEXT: v_or_b32_e32 v28, v28, v33
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v52
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v51
+; SI-NEXT: v_or_b32_e32 v7, v7, v55
+; SI-NEXT: v_or_b32_e32 v9, v9, v53
+; SI-NEXT: v_or_b32_e32 v11, v11, v50
+; SI-NEXT: v_or_b32_e32 v13, v13, v49
+; SI-NEXT: v_or_b32_e32 v15, v15, v48
+; SI-NEXT: v_or_b32_e32 v17, v17, v39
+; SI-NEXT: v_or_b32_e32 v19, v19, v38
+; SI-NEXT: v_or_b32_e32 v21, v21, v37
+; SI-NEXT: v_or_b32_e32 v23, v23, v36
+; SI-NEXT: v_or_b32_e32 v25, v25, v35
+; SI-NEXT: v_or_b32_e32 v27, v27, v34
+; SI-NEXT: v_or_b32_e32 v29, v29, v33
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -58840,23 +57540,23 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: v_writelane_b32 v63, s34, 0
; SI-NEXT: v_writelane_b32 v63, s35, 1
; SI-NEXT: v_writelane_b32 v63, s36, 2
@@ -58873,506 +57573,358 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: v_writelane_b32 v63, s55, 13
; SI-NEXT: v_writelane_b32 v63, s64, 14
; SI-NEXT: v_writelane_b32 v63, s65, 15
-; SI-NEXT: v_writelane_b32 v63, s66, 16
-; SI-NEXT: v_writelane_b32 v63, s67, 17
-; SI-NEXT: v_writelane_b32 v63, s68, 18
-; SI-NEXT: v_writelane_b32 v63, s69, 19
-; SI-NEXT: v_writelane_b32 v63, s70, 20
-; SI-NEXT: v_writelane_b32 v63, s71, 21
-; SI-NEXT: v_writelane_b32 v63, s80, 22
-; SI-NEXT: v_writelane_b32 v63, s81, 23
-; SI-NEXT: v_writelane_b32 v63, s82, 24
-; SI-NEXT: v_writelane_b32 v63, s83, 25
-; SI-NEXT: v_writelane_b32 v63, s84, 26
-; SI-NEXT: v_writelane_b32 v63, s85, 27
-; SI-NEXT: v_writelane_b32 v63, s86, 28
-; SI-NEXT: v_writelane_b32 v63, s87, 29
-; SI-NEXT: v_writelane_b32 v63, s96, 30
-; SI-NEXT: v_writelane_b32 v63, s97, 31
-; SI-NEXT: v_writelane_b32 v63, s98, 32
-; SI-NEXT: v_writelane_b32 v63, s99, 33
-; SI-NEXT: v_writelane_b32 v63, s30, 34
-; SI-NEXT: v_writelane_b32 v63, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: v_readfirstlane_b32 s7, v16
-; SI-NEXT: v_readfirstlane_b32 s8, v15
-; SI-NEXT: v_readfirstlane_b32 s9, v14
-; SI-NEXT: v_readfirstlane_b32 s10, v13
-; SI-NEXT: v_readfirstlane_b32 s11, v12
-; SI-NEXT: v_readfirstlane_b32 s12, v11
-; SI-NEXT: v_readfirstlane_b32 s13, v10
-; SI-NEXT: v_readfirstlane_b32 s14, v9
-; SI-NEXT: v_readfirstlane_b32 s15, v8
-; SI-NEXT: v_readfirstlane_b32 s40, v7
-; SI-NEXT: v_readfirstlane_b32 s41, v6
-; SI-NEXT: v_readfirstlane_b32 s42, v5
-; SI-NEXT: v_readfirstlane_b32 s43, v4
-; SI-NEXT: v_readfirstlane_b32 s44, v3
-; SI-NEXT: v_readfirstlane_b32 s45, v2
-; SI-NEXT: v_readfirstlane_b32 s46, v1
-; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
+; SI-NEXT: v_writelane_b32 v63, s30, 16
+; SI-NEXT: v_writelane_b32 v63, s31, 17
+; SI-NEXT: v_readfirstlane_b32 s44, v18
+; SI-NEXT: v_readfirstlane_b32 s5, v17
+; SI-NEXT: v_readfirstlane_b32 s4, v16
+; SI-NEXT: v_readfirstlane_b32 s7, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: v_readfirstlane_b32 s9, v13
+; SI-NEXT: v_readfirstlane_b32 s8, v12
+; SI-NEXT: v_readfirstlane_b32 s11, v11
+; SI-NEXT: v_readfirstlane_b32 s10, v10
+; SI-NEXT: v_readfirstlane_b32 s13, v9
+; SI-NEXT: v_readfirstlane_b32 s12, v8
+; SI-NEXT: v_readfirstlane_b32 s15, v7
+; SI-NEXT: v_readfirstlane_b32 s14, v6
+; SI-NEXT: v_readfirstlane_b32 s41, v5
+; SI-NEXT: v_readfirstlane_b32 s40, v4
+; SI-NEXT: v_readfirstlane_b32 s43, v3
+; SI-NEXT: v_readfirstlane_b32 s42, v2
+; SI-NEXT: v_readfirstlane_b32 s45, v1
+; SI-NEXT: s_cmp_lg_u32 s44, 0
+; SI-NEXT: v_readfirstlane_b32 s44, v0
; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s4, s6, 0xffff0000
-; SI-NEXT: v_writelane_b32 v62, s4, 1
-; SI-NEXT: s_lshl_b32 s4, s6, 16
-; SI-NEXT: v_writelane_b32 v62, s4, 0
-; SI-NEXT: s_and_b32 s58, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s7, 16
-; SI-NEXT: s_and_b32 s60, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s61, s8, 16
-; SI-NEXT: s_and_b32 s62, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s63, s9, 16
-; SI-NEXT: s_and_b32 s72, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s73, s10, 16
-; SI-NEXT: s_and_b32 s74, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s75, s11, 16
-; SI-NEXT: s_and_b32 s76, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s77, s12, 16
-; SI-NEXT: s_and_b32 s78, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s79, s13, 16
-; SI-NEXT: s_and_b32 s88, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s89, s14, 16
-; SI-NEXT: s_and_b32 s90, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s91, s15, 16
-; SI-NEXT: s_and_b32 s92, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s93, s40, 16
-; SI-NEXT: s_and_b32 s94, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s95, s41, 16
-; SI-NEXT: s_and_b32 vcc_lo, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_hi, s42, 16
-; SI-NEXT: s_and_b32 s30, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s31, s43, 16
-; SI-NEXT: s_and_b32 s34, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s35, s44, 16
-; SI-NEXT: s_and_b32 s36, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s37, s45, 16
-; SI-NEXT: s_and_b32 s38, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s39, s46, 16
-; SI-NEXT: s_and_b32 s48, s47, 0xffff0000
-; SI-NEXT: s_lshl_b32 s49, s47, 16
-; SI-NEXT: s_and_b32 s50, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s51, s29, 16
-; SI-NEXT: s_and_b32 s52, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s53, s28, 16
-; SI-NEXT: s_and_b32 s54, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s55, s27, 16
-; SI-NEXT: s_and_b32 s64, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s65, s26, 16
-; SI-NEXT: s_and_b32 s66, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s25, 16
-; SI-NEXT: s_and_b32 s68, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s69, s24, 16
-; SI-NEXT: s_and_b32 s70, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s23, 16
-; SI-NEXT: s_and_b32 s80, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s22, 16
-; SI-NEXT: s_and_b32 s82, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, s21, 16
-; SI-NEXT: s_and_b32 s84, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s20, 16
-; SI-NEXT: s_and_b32 s86, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s19, 16
-; SI-NEXT: s_and_b32 s96, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s97, s18, 16
-; SI-NEXT: s_and_b32 s98, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s99, s17, 16
-; SI-NEXT: s_and_b32 s56, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s56, s23, 16
+; SI-NEXT: s_lshr_b32 s57, s21, 16
+; SI-NEXT: s_lshr_b32 s58, s19, 16
+; SI-NEXT: s_lshr_b32 s59, s17, 16
+; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
+; SI-NEXT: s_lshr_b32 s65, s5, 16
+; SI-NEXT: s_lshr_b32 s64, s7, 16
+; SI-NEXT: s_lshr_b32 s55, s9, 16
+; SI-NEXT: s_lshr_b32 s54, s11, 16
+; SI-NEXT: s_lshr_b32 s53, s13, 16
+; SI-NEXT: s_lshr_b32 s52, s15, 16
+; SI-NEXT: s_lshr_b32 s51, s41, 16
+; SI-NEXT: s_lshr_b32 s50, s43, 16
+; SI-NEXT: s_lshr_b32 s49, s45, 16
+; SI-NEXT: s_lshr_b32 s48, s29, 16
+; SI-NEXT: s_lshr_b32 s39, s27, 16
+; SI-NEXT: s_lshr_b32 s38, s25, 16
+; SI-NEXT: s_mov_b32 s35, s56
+; SI-NEXT: s_mov_b32 s31, s57
+; SI-NEXT: s_mov_b32 vcc_hi, s58
+; SI-NEXT: s_mov_b32 s47, s59
+; SI-NEXT: s_lshr_b64 s[56:57], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
; SI-NEXT: s_branch .LBB41_3
; SI-NEXT: .LBB41_2:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: s_mov_b64 s[36:37], -1
; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr47
; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $vcc_hi
; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr31
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr35
; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr38
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr39
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr48
; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr49
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr51
; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: .LBB41_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s57, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s57, 1
; SI-NEXT: s_cbranch_scc1 .LBB41_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v29, s6, 1.0
-; SI-NEXT: v_add_f32_e64 v28, s7, 1.0
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_add_f32_e64 v27, s8, 1.0
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e64 v31, s5, 1.0
+; SI-NEXT: v_add_f32_e64 v30, s4, 1.0
+; SI-NEXT: v_lshr_b64 v[32:33], v[30:31], 16
+; SI-NEXT: v_add_f32_e64 v29, s7, 1.0
+; SI-NEXT: v_add_f32_e64 v28, s6, 1.0
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e64 v17, s43, 1.0
+; SI-NEXT: v_add_f32_e64 v16, s42, 1.0
+; SI-NEXT: v_add_f32_e64 v27, s9, 1.0
+; SI-NEXT: v_add_f32_e64 v26, s8, 1.0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_add_f32_e64 v0, s18, 1.0
-; SI-NEXT: v_add_f32_e64 v4, s22, 1.0
-; SI-NEXT: v_add_f32_e64 v5, s23, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s47, 1.0
-; SI-NEXT: v_add_f32_e64 v13, s46, 1.0
-; SI-NEXT: v_add_f32_e64 v14, s45, 1.0
-; SI-NEXT: v_add_f32_e64 v15, s44, 1.0
-; SI-NEXT: v_add_f32_e64 v26, s9, 1.0
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e64 v2, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v3, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v9, s27, 1.0
-; SI-NEXT: v_add_f32_e64 v10, s28, 1.0
-; SI-NEXT: v_add_f32_e64 v11, s29, 1.0
-; SI-NEXT: v_add_f32_e64 v22, s13, 1.0
-; SI-NEXT: v_add_f32_e64 v23, s12, 1.0
-; SI-NEXT: v_add_f32_e64 v24, s11, 1.0
-; SI-NEXT: v_add_f32_e64 v25, s10, 1.0
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v15
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v13
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v13
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v12
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v12
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v5
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_add_f32_e64 v0, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v6, s24, 1.0
-; SI-NEXT: v_add_f32_e64 v7, s25, 1.0
-; SI-NEXT: v_add_f32_e64 v8, s26, 1.0
-; SI-NEXT: v_add_f32_e64 v16, s43, 1.0
-; SI-NEXT: v_add_f32_e64 v17, s42, 1.0
-; SI-NEXT: v_add_f32_e64 v18, s41, 1.0
-; SI-NEXT: v_add_f32_e64 v19, s40, 1.0
-; SI-NEXT: v_add_f32_e64 v20, s15, 1.0
-; SI-NEXT: v_add_f32_e64 v21, s14, 1.0
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v25
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v24
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v23
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v22
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v11
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v11
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v9
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_lshr_b64 v[33:34], v[28:29], 16
+; SI-NEXT: v_add_f32_e64 v15, s45, 1.0
+; SI-NEXT: v_add_f32_e64 v14, s44, 1.0
+; SI-NEXT: v_add_f32_e64 v25, s11, 1.0
+; SI-NEXT: v_add_f32_e64 v24, s10, 1.0
+; SI-NEXT: v_lshr_b64 v[34:35], v[26:27], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[16:17], 16
+; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
+; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
+; SI-NEXT: v_add_f32_e64 v23, s13, 1.0
+; SI-NEXT: v_add_f32_e64 v22, s12, 1.0
+; SI-NEXT: v_lshr_b64 v[35:36], v[24:25], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[14:15], 16
+; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
+; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
+; SI-NEXT: v_add_f32_e64 v21, s15, 1.0
+; SI-NEXT: v_add_f32_e64 v20, s14, 1.0
+; SI-NEXT: v_lshr_b64 v[36:37], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[50:51], v[12:13], 16
+; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
+; SI-NEXT: v_add_f32_e64 v19, s41, 1.0
+; SI-NEXT: v_add_f32_e64 v18, s40, 1.0
+; SI-NEXT: v_lshr_b64 v[37:38], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[51:52], v[10:11], 16
+; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
+; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
+; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
+; SI-NEXT: v_lshr_b64 v[38:39], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[52:53], v[8:9], 16
+; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v21
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v20
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v19
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v18
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v17
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v8
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v6
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
+; SI-NEXT: v_lshr_b64 v[53:54], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[39:40], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[40:41], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
; SI-NEXT: s_branch .LBB41_6
; SI-NEXT: .LBB41_5:
-; SI-NEXT: v_mov_b32_e32 v7, s63
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s61
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s60
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s59
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s58
-; SI-NEXT: v_readlane_b32 s4, v62, 0
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 1
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_mov_b32_e32 v0, s57
-; SI-NEXT: v_mov_b32_e32 v1, s56
-; SI-NEXT: v_mov_b32_e32 v2, s99
-; SI-NEXT: v_mov_b32_e32 v3, s98
-; SI-NEXT: v_mov_b32_e32 v4, s97
-; SI-NEXT: v_mov_b32_e32 v5, s96
-; SI-NEXT: v_mov_b32_e32 v6, s87
-; SI-NEXT: v_mov_b32_e32 v39, s86
-; SI-NEXT: v_mov_b32_e32 v38, s85
-; SI-NEXT: v_mov_b32_e32 v9, s84
-; SI-NEXT: v_mov_b32_e32 v10, s83
-; SI-NEXT: v_mov_b32_e32 v11, s82
-; SI-NEXT: v_mov_b32_e32 v12, s81
-; SI-NEXT: v_mov_b32_e32 v13, s80
-; SI-NEXT: v_mov_b32_e32 v14, s71
-; SI-NEXT: v_mov_b32_e32 v15, s70
-; SI-NEXT: v_mov_b32_e32 v48, s69
-; SI-NEXT: v_mov_b32_e32 v17, s68
-; SI-NEXT: v_mov_b32_e32 v18, s67
-; SI-NEXT: v_mov_b32_e32 v19, s66
-; SI-NEXT: v_mov_b32_e32 v20, s65
-; SI-NEXT: v_mov_b32_e32 v21, s64
-; SI-NEXT: v_mov_b32_e32 v22, s55
-; SI-NEXT: v_mov_b32_e32 v23, s54
-; SI-NEXT: v_mov_b32_e32 v24, s53
-; SI-NEXT: v_mov_b32_e32 v25, s52
-; SI-NEXT: v_mov_b32_e32 v26, s51
-; SI-NEXT: v_mov_b32_e32 v27, s50
-; SI-NEXT: v_mov_b32_e32 v28, s49
-; SI-NEXT: v_mov_b32_e32 v29, s48
-; SI-NEXT: v_mov_b32_e32 v30, s39
-; SI-NEXT: v_mov_b32_e32 v31, s38
-; SI-NEXT: v_mov_b32_e32 v32, s37
-; SI-NEXT: v_mov_b32_e32 v33, s36
-; SI-NEXT: v_mov_b32_e32 v36, s35
-; SI-NEXT: v_mov_b32_e32 v37, s34
-; SI-NEXT: v_mov_b32_e32 v60, s31
-; SI-NEXT: v_mov_b32_e32 v61, s30
-; SI-NEXT: v_mov_b32_e32 v58, vcc_hi
-; SI-NEXT: v_mov_b32_e32 v59, vcc_lo
-; SI-NEXT: v_mov_b32_e32 v56, s95
-; SI-NEXT: v_mov_b32_e32 v57, s94
-; SI-NEXT: v_mov_b32_e32 v46, s93
-; SI-NEXT: v_mov_b32_e32 v47, s92
-; SI-NEXT: v_mov_b32_e32 v44, s91
-; SI-NEXT: v_mov_b32_e32 v45, s90
-; SI-NEXT: v_mov_b32_e32 v42, s89
-; SI-NEXT: v_mov_b32_e32 v43, s88
-; SI-NEXT: v_mov_b32_e32 v40, s79
-; SI-NEXT: v_mov_b32_e32 v41, s78
-; SI-NEXT: v_mov_b32_e32 v54, s77
-; SI-NEXT: v_mov_b32_e32 v55, s76
-; SI-NEXT: v_mov_b32_e32 v52, s75
-; SI-NEXT: v_mov_b32_e32 v53, s74
-; SI-NEXT: v_mov_b32_e32 v50, s73
-; SI-NEXT: v_mov_b32_e32 v51, s72
-; SI-NEXT: v_mov_b32_e32 v49, s62
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: .LBB41_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v33, vcc_lo
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v14, s44
+; SI-NEXT: v_mov_b32_e32 v15, s45
+; SI-NEXT: v_mov_b32_e32 v16, s42
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v18, s40
+; SI-NEXT: v_mov_b32_e32 v19, s41
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v21, s15
+; SI-NEXT: v_mov_b32_e32 v22, s12
+; SI-NEXT: v_mov_b32_e32 v23, s13
+; SI-NEXT: v_mov_b32_e32 v24, s10
+; SI-NEXT: v_mov_b32_e32 v25, s11
+; SI-NEXT: v_mov_b32_e32 v26, s8
+; SI-NEXT: v_mov_b32_e32 v27, s9
+; SI-NEXT: v_mov_b32_e32 v28, s6
+; SI-NEXT: v_mov_b32_e32 v29, s7
+; SI-NEXT: v_mov_b32_e32 v30, s4
+; SI-NEXT: v_mov_b32_e32 v31, s5
+; SI-NEXT: v_mov_b32_e32 v32, s47
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v62, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v61, s31
+; SI-NEXT: v_mov_b32_e32 v60, s35
+; SI-NEXT: v_mov_b32_e32 v59, s38
+; SI-NEXT: v_mov_b32_e32 v58, s39
+; SI-NEXT: v_mov_b32_e32 v57, s48
+; SI-NEXT: v_mov_b32_e32 v56, s49
+; SI-NEXT: v_mov_b32_e32 v47, s50
+; SI-NEXT: v_mov_b32_e32 v46, s51
+; SI-NEXT: v_mov_b32_e32 v45, s52
+; SI-NEXT: v_mov_b32_e32 v44, s53
+; SI-NEXT: v_mov_b32_e32 v43, s54
+; SI-NEXT: v_mov_b32_e32 v42, s55
+; SI-NEXT: v_mov_b32_e32 v41, s64
+; SI-NEXT: v_mov_b32_e32 v55, s65
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
+; SI-NEXT: v_mov_b32_e32 v33, s30
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[33:34], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v38
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v60
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v46
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v44
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v42
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v50
-; SI-NEXT: v_readlane_b32 s30, v63, 34
-; SI-NEXT: v_readlane_b32 s31, v63, 35
-; SI-NEXT: v_readlane_b32 s99, v63, 33
-; SI-NEXT: v_readlane_b32 s98, v63, 32
-; SI-NEXT: v_readlane_b32 s97, v63, 31
-; SI-NEXT: v_readlane_b32 s96, v63, 30
-; SI-NEXT: v_readlane_b32 s87, v63, 29
-; SI-NEXT: v_readlane_b32 s86, v63, 28
-; SI-NEXT: v_readlane_b32 s85, v63, 27
-; SI-NEXT: v_readlane_b32 s84, v63, 26
-; SI-NEXT: v_readlane_b32 s83, v63, 25
-; SI-NEXT: v_readlane_b32 s82, v63, 24
-; SI-NEXT: v_readlane_b32 s81, v63, 23
-; SI-NEXT: v_readlane_b32 s80, v63, 22
-; SI-NEXT: v_readlane_b32 s71, v63, 21
-; SI-NEXT: v_readlane_b32 s70, v63, 20
-; SI-NEXT: v_readlane_b32 s69, v63, 19
-; SI-NEXT: v_readlane_b32 s68, v63, 18
-; SI-NEXT: v_readlane_b32 s67, v63, 17
-; SI-NEXT: v_readlane_b32 s66, v63, 16
+; SI-NEXT: v_mov_b32_e32 v34, s34
+; SI-NEXT: v_mov_b32_e32 v35, s46
+; SI-NEXT: v_mov_b32_e32 v36, s56
+; SI-NEXT: v_mov_b32_e32 v37, s58
+; SI-NEXT: v_mov_b32_e32 v38, s60
+; SI-NEXT: v_mov_b32_e32 v48, s62
+; SI-NEXT: v_mov_b32_e32 v49, s72
+; SI-NEXT: v_mov_b32_e32 v50, s74
+; SI-NEXT: v_mov_b32_e32 v51, s76
+; SI-NEXT: v_mov_b32_e32 v52, s78
+; SI-NEXT: v_mov_b32_e32 v53, s88
+; SI-NEXT: v_mov_b32_e32 v54, s90
+; SI-NEXT: v_mov_b32_e32 v39, s92
+; SI-NEXT: v_mov_b32_e32 v40, s94
+; SI-NEXT: .LBB41_6: ; %end
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_or_b32_e32 v1, v1, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v39
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v2, v2, v32
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v62
+; SI-NEXT: v_or_b32_e32 v3, v3, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v54
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v4, v4, v32
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v61
+; SI-NEXT: v_or_b32_e32 v5, v5, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v53
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v6, v6, v32
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v60
+; SI-NEXT: v_or_b32_e32 v7, v7, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v52
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v8, v8, v32
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v59
+; SI-NEXT: v_or_b32_e32 v9, v9, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v51
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v10, v10, v32
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v58
+; SI-NEXT: v_or_b32_e32 v11, v11, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v50
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v32
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v57
+; SI-NEXT: v_or_b32_e32 v13, v13, v32
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v49
+; SI-NEXT: v_or_b32_e32 v14, v14, v32
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v56
+; SI-NEXT: v_or_b32_e32 v15, v15, v32
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v48
+; SI-NEXT: v_or_b32_e32 v16, v16, v32
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v47
+; SI-NEXT: v_or_b32_e32 v17, v17, v32
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v38
+; SI-NEXT: v_or_b32_e32 v18, v18, v32
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v46
+; SI-NEXT: v_or_b32_e32 v19, v19, v32
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; SI-NEXT: v_or_b32_e32 v20, v20, v32
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v45
+; SI-NEXT: v_or_b32_e32 v21, v21, v32
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v36
+; SI-NEXT: v_or_b32_e32 v22, v22, v32
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v44
+; SI-NEXT: v_or_b32_e32 v23, v23, v32
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v35
+; SI-NEXT: v_or_b32_e32 v24, v24, v32
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v43
+; SI-NEXT: v_or_b32_e32 v25, v25, v32
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v34
+; SI-NEXT: v_or_b32_e32 v26, v26, v32
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v42
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v27, v27, v32
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
+; SI-NEXT: v_or_b32_e32 v0, v0, v40
+; SI-NEXT: v_or_b32_e32 v28, v28, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v41
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_readlane_b32 s30, v63, 16
+; SI-NEXT: v_readlane_b32 s31, v63, 17
; SI-NEXT: v_readlane_b32 s65, v63, 15
; SI-NEXT: v_readlane_b32 s64, v63, 14
; SI-NEXT: v_readlane_b32 s55, v63, 13
@@ -59389,85 +57941,13 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[4:5], v[5:6], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[24:25], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[26:27], 16
-; SI-NEXT: v_lshr_b64 v[14:15], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[30:31], 16
-; SI-NEXT: v_lshr_b64 v[16:17], v[17:18], 16
-; SI-NEXT: v_lshr_b64 v[17:18], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[60:61], 16
-; SI-NEXT: v_lshr_b64 v[19:20], v[58:59], 16
-; SI-NEXT: v_lshr_b64 v[20:21], v[56:57], 16
-; SI-NEXT: v_lshr_b64 v[21:22], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[42:43], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[40:41], 16
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[25:26], v[54:55], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[38:39], 16
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[30:31], v[36:37], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[31:32], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v0, v33
-; SI-NEXT: v_mov_b32_e32 v1, v34
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v55
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -59888,265 +58368,308 @@ define <32 x float> @bitcast_v64bf16_to_v32f32(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v62, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v63
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32
+; SI-NEXT: v_mov_b32_e32 v44, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; SI-NEXT: v_mov_b32_e32 v45, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v19
+; SI-NEXT: v_mov_b32_e32 v46, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v20
+; SI-NEXT: v_mov_b32_e32 v47, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
+; SI-NEXT: v_mov_b32_e32 v56, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_mov_b32_e32 v57, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v23
+; SI-NEXT: v_mov_b32_e32 v58, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v24
+; SI-NEXT: v_mov_b32_e32 v59, v3
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v25
+; SI-NEXT: v_mov_b32_e32 v60, v2
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v26
+; SI-NEXT: v_mov_b32_e32 v61, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v27
+; SI-NEXT: v_mov_b32_e32 v42, v12
+; SI-NEXT: v_mov_b32_e32 v43, v11
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v17
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v62
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v42
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v43
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v45
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v46
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v56
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v57
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v58
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v59
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v60
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v61
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v23
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v11
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v53
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v37
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v38
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v38
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v7
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v63
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v51
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v35
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v34
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB42_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v3, v3, v32
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v62
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v60
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v58
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v56
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v47
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v46
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v45
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v44
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v43
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: v_or_b32_e32 v0, v0, v41
+; SI-NEXT: v_or_b32_e32 v1, v1, v48
+; SI-NEXT: v_or_b32_e32 v2, v2, v40
+; SI-NEXT: v_or_b32_e32 v4, v4, v55
+; SI-NEXT: v_or_b32_e32 v5, v5, v39
+; SI-NEXT: v_or_b32_e32 v6, v6, v54
+; SI-NEXT: v_or_b32_e32 v7, v7, v35
+; SI-NEXT: v_or_b32_e32 v8, v8, v53
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v34
+; SI-NEXT: v_or_b32_e32 v12, v12, v51
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v18, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v13, v13, v37
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v15, v15, v33
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v16, v16, v49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v17, v17, v36
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v63
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
@@ -60180,393 +58703,238 @@ define <32 x float> @bitcast_v64bf16_to_v32f32(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v43
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v34
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_alignbit_b32 v19, v19, v59, 16
-; SI-NEXT: v_alignbit_b32 v20, v20, v57, 16
-; SI-NEXT: v_alignbit_b32 v21, v21, v46, 16
-; SI-NEXT: v_alignbit_b32 v22, v22, v44, 16
-; SI-NEXT: v_alignbit_b32 v23, v23, v42, 16
-; SI-NEXT: v_alignbit_b32 v24, v24, v40, 16
-; SI-NEXT: v_alignbit_b32 v25, v25, v54, 16
-; SI-NEXT: v_alignbit_b32 v26, v26, v52, 16
-; SI-NEXT: v_alignbit_b32 v27, v27, v50, 16
-; SI-NEXT: v_alignbit_b32 v28, v28, v48, 16
-; SI-NEXT: v_alignbit_b32 v29, v29, v38, 16
-; SI-NEXT: v_alignbit_b32 v30, v30, v35, 16
-; SI-NEXT: v_alignbit_b32 v31, v31, v33, 16
; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: .LBB42_2: ; %Flow
+; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB42_4
+; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v41
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v61
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v48
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_alignbit_b32 v0, v0, v63, 16
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v60
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v59
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v32
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v58
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v55
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v57
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v39
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v54
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v47
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v35
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v46
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v45
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v44
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v43
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v34
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v42
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v51
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v37
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v50
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v62
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v60
-; SI-NEXT: v_alignbit_b32 v18, v18, v61, 16
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: .LBB42_2: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB42_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v49
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v36
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v60
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v58
+; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v47
+; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v45
+; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v43
+; SI-NEXT: s_waitcnt vmcnt(10)
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v41
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v55
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v53
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v51
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v49
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v39
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v36
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v34
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v32
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_alignbit_b32 v15, v16, v15, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_alignbit_b32 v16, v17, v16, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_alignbit_b32 v17, v18, v17, 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v61
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_alignbit_b32 v18, v19, v18, 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v59
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_alignbit_b32 v19, v20, v19, 16
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v57
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_alignbit_b32 v20, v21, v20, 16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v46
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_alignbit_b32 v21, v22, v21, 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v44
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_alignbit_b32 v22, v23, v22, 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v42
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_alignbit_b32 v23, v24, v23, 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_alignbit_b32 v24, v25, v24, 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v54
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_alignbit_b32 v25, v26, v25, 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v52
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_alignbit_b32 v26, v27, v26, 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v50
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_alignbit_b32 v27, v28, v27, 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v48
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_alignbit_b32 v28, v29, v28, 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v38
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_alignbit_b32 v29, v30, v29, 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v35
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_alignbit_b32 v30, v31, v30, 16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v33
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v63
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_alignbit_b32 v31, v32, v31, 16
; SI-NEXT: .LBB42_4: ; %end
@@ -62922,768 +61290,486 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-LABEL: bitcast_v64bf16_to_v32f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_writelane_b32 v63, s34, 0
-; SI-NEXT: v_writelane_b32 v63, s35, 1
-; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s30, 3
-; SI-NEXT: v_writelane_b32 v63, s31, 4
-; SI-NEXT: v_readfirstlane_b32 s78, v7
-; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s72, v10
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
-; SI-NEXT: v_readfirstlane_b32 s62, v11
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
-; SI-NEXT: v_readfirstlane_b32 s60, v12
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
-; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
-; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s94
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s92, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s92
-; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s90
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v18
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s42
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s78
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v36, s34, 0
+; SI-NEXT: v_writelane_b32 v36, s35, 1
+; SI-NEXT: v_writelane_b32 v36, s36, 2
+; SI-NEXT: v_writelane_b32 v36, s37, 3
+; SI-NEXT: v_writelane_b32 v36, s38, 4
+; SI-NEXT: v_writelane_b32 v36, s39, 5
+; SI-NEXT: v_writelane_b32 v36, s48, 6
+; SI-NEXT: v_writelane_b32 v36, s49, 7
+; SI-NEXT: v_writelane_b32 v36, s50, 8
+; SI-NEXT: v_writelane_b32 v36, s51, 9
+; SI-NEXT: v_writelane_b32 v36, s52, 10
+; SI-NEXT: v_writelane_b32 v36, s53, 11
+; SI-NEXT: v_writelane_b32 v36, s54, 12
+; SI-NEXT: v_writelane_b32 v36, s55, 13
+; SI-NEXT: v_writelane_b32 v36, s64, 14
+; SI-NEXT: v_writelane_b32 v36, s65, 15
+; SI-NEXT: v_writelane_b32 v36, s66, 16
+; SI-NEXT: v_writelane_b32 v36, s67, 17
+; SI-NEXT: v_writelane_b32 v36, s68, 18
+; SI-NEXT: v_writelane_b32 v36, s69, 19
+; SI-NEXT: v_writelane_b32 v36, s70, 20
+; SI-NEXT: v_writelane_b32 v36, s71, 21
+; SI-NEXT: v_writelane_b32 v36, s80, 22
+; SI-NEXT: v_writelane_b32 v36, s81, 23
+; SI-NEXT: v_writelane_b32 v36, s82, 24
+; SI-NEXT: v_writelane_b32 v36, s83, 25
+; SI-NEXT: v_writelane_b32 v36, s84, 26
+; SI-NEXT: v_writelane_b32 v36, s85, 27
+; SI-NEXT: v_writelane_b32 v36, s86, 28
+; SI-NEXT: v_writelane_b32 v36, s87, 29
+; SI-NEXT: v_writelane_b32 v36, s96, 30
+; SI-NEXT: v_writelane_b32 v36, s97, 31
+; SI-NEXT: v_writelane_b32 v36, s98, 32
+; SI-NEXT: v_writelane_b32 v36, s99, 33
+; SI-NEXT: v_writelane_b32 v36, s30, 34
+; SI-NEXT: v_writelane_b32 v36, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
+; SI-NEXT: v_readfirstlane_b32 s11, v15
+; SI-NEXT: v_readfirstlane_b32 s13, v14
+; SI-NEXT: v_readfirstlane_b32 s15, v13
+; SI-NEXT: v_readfirstlane_b32 s73, v12
+; SI-NEXT: v_readfirstlane_b32 s75, v11
+; SI-NEXT: v_readfirstlane_b32 s77, v10
+; SI-NEXT: v_readfirstlane_b32 s79, v9
+; SI-NEXT: v_readfirstlane_b32 s89, v8
+; SI-NEXT: v_readfirstlane_b32 s91, v7
+; SI-NEXT: v_readfirstlane_b32 s93, v6
+; SI-NEXT: v_readfirstlane_b32 s95, v5
+; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s34, v3
+; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: v_readfirstlane_b32 s80, v1
+; SI-NEXT: v_readfirstlane_b32 s83, v0
+; SI-NEXT: s_lshr_b32 vcc_lo, s29, 16
+; SI-NEXT: s_lshr_b32 s30, s28, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s70, s26, 16
+; SI-NEXT: s_lshr_b32 s81, s25, 16
+; SI-NEXT: s_lshr_b32 s84, s24, 16
+; SI-NEXT: s_lshr_b32 s85, s23, 16
+; SI-NEXT: s_lshr_b32 s86, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s87, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s66, s9, 16
+; SI-NEXT: s_lshr_b32 s10, s11, 16
+; SI-NEXT: s_lshr_b32 s12, s13, 16
+; SI-NEXT: s_lshr_b32 s14, s15, 16
+; SI-NEXT: s_lshr_b32 s72, s73, 16
+; SI-NEXT: s_lshr_b32 s74, s75, 16
+; SI-NEXT: s_lshr_b32 s76, s77, 16
+; SI-NEXT: s_lshr_b32 s78, s79, 16
+; SI-NEXT: s_lshr_b32 s88, s89, 16
+; SI-NEXT: s_lshr_b32 s90, s91, 16
+; SI-NEXT: s_lshr_b32 s92, s93, 16
+; SI-NEXT: s_lshr_b32 s94, s95, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s4, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s69, 16
+; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 s82, s83, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: ; implicit-def: $vgpr37 : SGPR spill to VGPR lane
+; SI-NEXT: s_cmp_lg_u32 s5, 0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v37, s67, 0
+; SI-NEXT: v_writelane_b32 v37, s66, 1
; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v37
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v48
-; SI-NEXT: v_mov_b32_e32 v62, v35
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v18
-; SI-NEXT: v_mov_b32_e32 v29, v54
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v30, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s36, s5, s6
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s99, 16
+; SI-NEXT: s_or_b32 s38, s5, s6
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s98, 16
+; SI-NEXT: s_and_b32 s40, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s41, s87, 16
+; SI-NEXT: s_or_b32 s39, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s97, 16
+; SI-NEXT: s_or_b32 s37, s40, s41
+; SI-NEXT: s_or_b32 s40, s5, s6
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s96, 16
+; SI-NEXT: s_or_b32 s41, s5, s6
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s86, 16
+; SI-NEXT: s_or_b32 s42, s5, s6
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s85, 16
+; SI-NEXT: s_or_b32 s43, s5, s6
+; SI-NEXT: s_and_b32 s5, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s84, 16
+; SI-NEXT: s_or_b32 s44, s5, s6
+; SI-NEXT: s_and_b32 s5, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s81, 16
+; SI-NEXT: s_or_b32 s45, s5, s6
+; SI-NEXT: s_and_b32 s5, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s70, 16
+; SI-NEXT: s_or_b32 s46, s5, s6
+; SI-NEXT: s_and_b32 s5, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s35, 16
+; SI-NEXT: s_or_b32 s47, s5, s6
+; SI-NEXT: s_and_b32 s5, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s30, 16
+; SI-NEXT: s_or_b32 s48, s5, s6
+; SI-NEXT: s_and_b32 s5, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s6, vcc_lo, 16
+; SI-NEXT: s_or_b32 s49, s5, s6
+; SI-NEXT: s_and_b32 s5, s83, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s82, 16
+; SI-NEXT: s_or_b32 s50, s5, s6
+; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s71, 16
+; SI-NEXT: s_or_b32 s51, s5, s6
+; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s68, 16
+; SI-NEXT: s_or_b32 s52, s5, s6
+; SI-NEXT: s_and_b32 s5, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s31, 16
+; SI-NEXT: s_or_b32 s53, s5, s6
+; SI-NEXT: s_mov_b32 s6, s4
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_or_b32 s54, s4, s5
+; SI-NEXT: s_and_b32 s4, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_or_b32 s55, s4, s5
+; SI-NEXT: s_and_b32 s4, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_or_b32 s56, s4, s5
+; SI-NEXT: s_and_b32 s4, s91, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_or_b32 s57, s4, s5
+; SI-NEXT: s_and_b32 s4, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_or_b32 s58, s4, s5
+; SI-NEXT: s_and_b32 s4, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: s_or_b32 s59, s4, s5
+; SI-NEXT: s_and_b32 s4, s77, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_or_b32 s60, s4, s5
+; SI-NEXT: s_and_b32 s4, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_or_b32 s61, s4, s5
+; SI-NEXT: s_and_b32 s4, s73, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_or_b32 s62, s4, s5
+; SI-NEXT: s_and_b32 s4, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s63, s4, s5
+; SI-NEXT: s_and_b32 s4, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s64, s4, s5
+; SI-NEXT: s_and_b32 s4, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s10, 16
+; SI-NEXT: s_or_b32 s65, s4, s5
+; SI-NEXT: s_and_b32 s4, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: s_or_b32 s66, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_or_b32 s67, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(2) expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[38:39], 16
-; SI-NEXT: v_mov_b32_e32 v39, v33
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v39
-; SI-NEXT: v_lshr_b64 v[3:4], v[32:33], 16
-; SI-NEXT: v_mov_b32_e32 v4, v41
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[41:42], 16
-; SI-NEXT: v_mov_b32_e32 v5, v26
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[26:27], 16
-; SI-NEXT: v_mov_b32_e32 v6, v25
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v7, v24
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v8
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[24:25], 16
-; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v9, v13
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v33, v61
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v33
-; SI-NEXT: v_lshr_b64 v[10:11], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v13, v60
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[59:60], 16
-; SI-NEXT: v_mov_b32_e32 v14, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v15
-; SI-NEXT: v_mov_b32_e32 v60, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[58:59], 16
-; SI-NEXT: v_mov_b32_e32 v15, v58
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v16
-; SI-NEXT: v_mov_b32_e32 v59, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[57:58], 16
-; SI-NEXT: v_mov_b32_e32 v16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v17
-; SI-NEXT: v_mov_b32_e32 v58, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v17, v56
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v19
-; SI-NEXT: v_mov_b32_e32 v57, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[55:56], 16
-; SI-NEXT: v_mov_b32_e32 v56, v17
-; SI-NEXT: v_lshr_b64 v[16:17], v[51:52], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v20
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v42, v49
-; SI-NEXT: v_mov_b32_e32 v49, v44
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v27
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v54
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v25
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v21
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v22
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v50
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v26
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v24, v53
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[53:54], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v54, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v31
-; SI-NEXT: v_mov_b32_e32 v31, v50
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
-; SI-NEXT: v_lshr_b64 v[25:26], v[40:41], 16
-; SI-NEXT: v_mov_b32_e32 v26, v43
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[26:27], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v27, v28
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v37
-; SI-NEXT: v_mov_b32_e32 v37, v48
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v35
-; SI-NEXT: v_mov_b32_e32 v35, v62
-; SI-NEXT: v_lshr_b64 v[61:62], v[47:48], 16
-; SI-NEXT: v_mov_b32_e32 v44, v49
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[45:46], 16
-; SI-NEXT: v_mov_b32_e32 v46, v30
-; SI-NEXT: v_lshr_b64 v[29:30], v[49:50], 16
-; SI-NEXT: v_mov_b32_e32 v50, v31
-; SI-NEXT: v_lshr_b64 v[30:31], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v61, v33
-; SI-NEXT: v_mov_b32_e32 v33, v39
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v49, v42
; SI-NEXT: s_branch .LBB43_3
; SI-NEXT: .LBB43_2:
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b32 s6, s4
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
; SI-NEXT: .LBB43_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB43_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v31, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v35
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s87, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s99, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s98, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s97, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s96, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s86, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s85, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s84, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s81, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s70, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s35, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_lo, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: s_lshl_b32 s4, s83, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s82, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_lshl_b32 s4, s80, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s71, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: s_lshl_b32 s4, s69, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s68, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: s_lshl_b32 s4, s34, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s31, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: s_lshl_b32 s4, s95, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s94, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: s_lshl_b32 s4, s93, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s92, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_lshl_b32 s4, s91, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s90, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: s_lshl_b32 s4, s89, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s88, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: s_lshl_b32 s4, s79, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s78, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s77, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s76, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s74, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s72, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_add_f32_e32 v32, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 1
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v31, v32
-; SI-NEXT: .LBB43_5: ; %end
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s30, v63, 3
-; SI-NEXT: v_readlane_b32 s31, v63, 4
-; SI-NEXT: v_readlane_b32 s36, v63, 2
-; SI-NEXT: v_readlane_b32 s35, v63, 1
-; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 0
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v31, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_lshr_b64 v[34:35], v[30:31], 16
+; SI-NEXT: v_lshr_b64 v[30:31], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v31, v34
+; SI-NEXT: s_branch .LBB43_6
+; SI-NEXT: .LBB43_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: v_mov_b32_e32 v16, s52
+; SI-NEXT: v_mov_b32_e32 v17, s53
+; SI-NEXT: v_mov_b32_e32 v18, s54
+; SI-NEXT: v_mov_b32_e32 v19, s55
+; SI-NEXT: v_mov_b32_e32 v20, s56
+; SI-NEXT: v_mov_b32_e32 v21, s57
+; SI-NEXT: v_mov_b32_e32 v22, s58
+; SI-NEXT: v_mov_b32_e32 v23, s59
+; SI-NEXT: v_mov_b32_e32 v24, s60
+; SI-NEXT: v_mov_b32_e32 v25, s61
+; SI-NEXT: v_mov_b32_e32 v26, s62
+; SI-NEXT: v_mov_b32_e32 v27, s63
+; SI-NEXT: v_mov_b32_e32 v28, s64
+; SI-NEXT: v_mov_b32_e32 v29, s65
+; SI-NEXT: v_mov_b32_e32 v30, s66
+; SI-NEXT: v_mov_b32_e32 v31, s67
+; SI-NEXT: .LBB43_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v36, 34
+; SI-NEXT: v_readlane_b32 s31, v36, 35
+; SI-NEXT: v_readlane_b32 s99, v36, 33
+; SI-NEXT: v_readlane_b32 s98, v36, 32
+; SI-NEXT: v_readlane_b32 s97, v36, 31
+; SI-NEXT: v_readlane_b32 s96, v36, 30
+; SI-NEXT: v_readlane_b32 s87, v36, 29
+; SI-NEXT: v_readlane_b32 s86, v36, 28
+; SI-NEXT: v_readlane_b32 s85, v36, 27
+; SI-NEXT: v_readlane_b32 s84, v36, 26
+; SI-NEXT: v_readlane_b32 s83, v36, 25
+; SI-NEXT: v_readlane_b32 s82, v36, 24
+; SI-NEXT: v_readlane_b32 s81, v36, 23
+; SI-NEXT: v_readlane_b32 s80, v36, 22
+; SI-NEXT: v_readlane_b32 s71, v36, 21
+; SI-NEXT: v_readlane_b32 s70, v36, 20
+; SI-NEXT: v_readlane_b32 s69, v36, 19
+; SI-NEXT: v_readlane_b32 s68, v36, 18
+; SI-NEXT: v_readlane_b32 s67, v36, 17
+; SI-NEXT: v_readlane_b32 s66, v36, 16
+; SI-NEXT: v_readlane_b32 s65, v36, 15
+; SI-NEXT: v_readlane_b32 s64, v36, 14
+; SI-NEXT: v_readlane_b32 s55, v36, 13
+; SI-NEXT: v_readlane_b32 s54, v36, 12
+; SI-NEXT: v_readlane_b32 s53, v36, 11
+; SI-NEXT: v_readlane_b32 s52, v36, 10
+; SI-NEXT: v_readlane_b32 s51, v36, 9
+; SI-NEXT: v_readlane_b32 s50, v36, 8
+; SI-NEXT: v_readlane_b32 s49, v36, 7
+; SI-NEXT: v_readlane_b32 s48, v36, 6
+; SI-NEXT: v_readlane_b32 s39, v36, 5
+; SI-NEXT: v_readlane_b32 s38, v36, 4
+; SI-NEXT: v_readlane_b32 s37, v36, 3
+; SI-NEXT: v_readlane_b32 s36, v36, 2
+; SI-NEXT: v_readlane_b32 s35, v36, 1
+; SI-NEXT: v_readlane_b32 s34, v36, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64bf16_to_v32f32_scalar:
@@ -92407,255 +90493,86 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr62
; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr60
; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr59
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr58
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; kill: killed $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB60_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v62
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v52, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v54, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v41, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v44, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v47, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v62
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v13
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v12
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v11
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v10
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v9
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v8
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v7
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v2
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v1
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB60_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB60_4
@@ -92675,7 +90592,6 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
; SI-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_addc_u32_e32 v15, vcc, 0, v15, vcc
; SI-NEXT: v_add_i32_e32 v16, vcc, 3, v16
; SI-NEXT: v_addc_u32_e32 v17, vcc, 0, v17, vcc
@@ -92693,310 +90609,107 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; SI-NEXT: v_addc_u32_e32 v29, vcc, 0, v29, vcc
; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v30
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_addc_u32_e32 v31, vcc, 0, v62, vcc
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_addc_u32_e32 v31, vcc, 0, v31, vcc
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v52, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v54, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v41, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v44, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v47, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v13
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v12
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v11
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v10
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v9
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v8
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v7
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v2
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v1
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB60_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v60
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v58
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v56
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v46
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v44
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v42
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v40
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; SI-NEXT: v_or_b32_e32 v0, v0, v47
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v63
+; SI-NEXT: v_or_b32_e32 v2, v2, v44
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v62
+; SI-NEXT: v_or_b32_e32 v4, v4, v41
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v61
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_or_b32_e32 v1, v1, v47
+; SI-NEXT: v_or_b32_e32 v3, v3, v44
+; SI-NEXT: v_or_b32_e32 v5, v5, v41
+; SI-NEXT: v_or_b32_e32 v6, v6, v54
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v60
+; SI-NEXT: v_or_b32_e32 v8, v8, v52
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v59
+; SI-NEXT: v_or_b32_e32 v10, v10, v50
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v58
+; SI-NEXT: v_or_b32_e32 v12, v12, v49
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v57
+; SI-NEXT: v_or_b32_e32 v14, v14, v48
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v56
+; SI-NEXT: v_or_b32_e32 v16, v16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v46
+; SI-NEXT: v_or_b32_e32 v18, v18, v38
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v45
+; SI-NEXT: v_or_b32_e32 v20, v20, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v43
+; SI-NEXT: v_or_b32_e32 v22, v22, v36
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v42
+; SI-NEXT: v_or_b32_e32 v24, v24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v40
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v63
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@@ -93013,8 +90726,45 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v26, v26, v34
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v55
+; SI-NEXT: v_or_b32_e32 v28, v28, v33
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v53
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v51
+; SI-NEXT: v_or_b32_e32 v7, v7, v54
+; SI-NEXT: v_or_b32_e32 v9, v9, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v50
+; SI-NEXT: v_or_b32_e32 v13, v13, v49
+; SI-NEXT: v_or_b32_e32 v15, v15, v48
+; SI-NEXT: v_or_b32_e32 v17, v17, v39
+; SI-NEXT: v_or_b32_e32 v19, v19, v38
+; SI-NEXT: v_or_b32_e32 v21, v21, v37
+; SI-NEXT: v_or_b32_e32 v23, v23, v36
+; SI-NEXT: v_or_b32_e32 v25, v25, v35
+; SI-NEXT: v_or_b32_e32 v27, v27, v34
+; SI-NEXT: v_or_b32_e32 v29, v29, v33
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -93199,574 +90949,385 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_writelane_b32 v33, s34, 0
-; SI-NEXT: v_writelane_b32 v33, s35, 1
-; SI-NEXT: v_writelane_b32 v33, s36, 2
-; SI-NEXT: v_writelane_b32 v33, s37, 3
-; SI-NEXT: v_writelane_b32 v33, s38, 4
-; SI-NEXT: v_writelane_b32 v33, s39, 5
-; SI-NEXT: v_writelane_b32 v33, s48, 6
-; SI-NEXT: v_writelane_b32 v33, s49, 7
-; SI-NEXT: v_writelane_b32 v33, s50, 8
-; SI-NEXT: v_writelane_b32 v33, s51, 9
-; SI-NEXT: v_writelane_b32 v33, s52, 10
-; SI-NEXT: v_writelane_b32 v33, s53, 11
-; SI-NEXT: v_writelane_b32 v33, s54, 12
-; SI-NEXT: v_writelane_b32 v33, s55, 13
-; SI-NEXT: v_writelane_b32 v33, s64, 14
-; SI-NEXT: v_writelane_b32 v33, s65, 15
-; SI-NEXT: v_writelane_b32 v33, s66, 16
-; SI-NEXT: v_writelane_b32 v33, s67, 17
-; SI-NEXT: v_writelane_b32 v33, s68, 18
-; SI-NEXT: v_writelane_b32 v33, s69, 19
-; SI-NEXT: v_writelane_b32 v33, s70, 20
-; SI-NEXT: v_writelane_b32 v33, s71, 21
-; SI-NEXT: v_writelane_b32 v33, s80, 22
-; SI-NEXT: v_writelane_b32 v33, s81, 23
-; SI-NEXT: v_writelane_b32 v33, s82, 24
-; SI-NEXT: v_writelane_b32 v33, s83, 25
-; SI-NEXT: v_writelane_b32 v33, s84, 26
-; SI-NEXT: v_writelane_b32 v33, s85, 27
-; SI-NEXT: v_writelane_b32 v33, s86, 28
-; SI-NEXT: v_writelane_b32 v33, s87, 29
-; SI-NEXT: v_writelane_b32 v33, s96, 30
-; SI-NEXT: v_writelane_b32 v33, s97, 31
-; SI-NEXT: v_writelane_b32 v33, s98, 32
-; SI-NEXT: v_writelane_b32 v33, s30, 33
-; SI-NEXT: v_writelane_b32 v33, s31, 34
-; SI-NEXT: v_readfirstlane_b32 s5, v18
-; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: v_readfirstlane_b32 s30, v16
-; SI-NEXT: v_readfirstlane_b32 s31, v15
-; SI-NEXT: v_readfirstlane_b32 s34, v14
-; SI-NEXT: v_readfirstlane_b32 s35, v13
-; SI-NEXT: v_readfirstlane_b32 s36, v12
-; SI-NEXT: v_readfirstlane_b32 s37, v11
-; SI-NEXT: v_readfirstlane_b32 s38, v10
-; SI-NEXT: v_readfirstlane_b32 s39, v9
-; SI-NEXT: v_readfirstlane_b32 s48, v8
-; SI-NEXT: v_readfirstlane_b32 s49, v7
-; SI-NEXT: v_readfirstlane_b32 s50, v6
-; SI-NEXT: v_readfirstlane_b32 s51, v5
-; SI-NEXT: v_readfirstlane_b32 s52, v4
-; SI-NEXT: v_readfirstlane_b32 s53, v3
-; SI-NEXT: v_readfirstlane_b32 s54, v2
-; SI-NEXT: v_readfirstlane_b32 s55, v1
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_readfirstlane_b32 s64, v0
-; SI-NEXT: ; implicit-def: $vgpr34 : SGPR spill to VGPR lane
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v34, s4, 0
+; SI-NEXT: v_writelane_b32 v32, s34, 0
+; SI-NEXT: v_writelane_b32 v32, s35, 1
+; SI-NEXT: v_writelane_b32 v32, s36, 2
+; SI-NEXT: v_writelane_b32 v32, s37, 3
+; SI-NEXT: v_writelane_b32 v32, s38, 4
+; SI-NEXT: v_writelane_b32 v32, s39, 5
+; SI-NEXT: v_writelane_b32 v32, s48, 6
+; SI-NEXT: v_writelane_b32 v32, s49, 7
+; SI-NEXT: v_writelane_b32 v32, s50, 8
+; SI-NEXT: v_writelane_b32 v32, s51, 9
+; SI-NEXT: v_writelane_b32 v32, s30, 10
+; SI-NEXT: v_writelane_b32 v32, s31, 11
+; SI-NEXT: v_readfirstlane_b32 s44, v18
+; SI-NEXT: v_readfirstlane_b32 s5, v17
+; SI-NEXT: v_readfirstlane_b32 s4, v16
+; SI-NEXT: v_readfirstlane_b32 s7, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: v_readfirstlane_b32 s9, v13
+; SI-NEXT: v_readfirstlane_b32 s8, v12
+; SI-NEXT: v_readfirstlane_b32 s11, v11
+; SI-NEXT: v_readfirstlane_b32 s10, v10
+; SI-NEXT: v_readfirstlane_b32 s13, v9
+; SI-NEXT: v_readfirstlane_b32 s12, v8
+; SI-NEXT: v_readfirstlane_b32 s15, v7
+; SI-NEXT: v_readfirstlane_b32 s14, v6
+; SI-NEXT: v_readfirstlane_b32 s41, v5
+; SI-NEXT: v_readfirstlane_b32 s40, v4
+; SI-NEXT: v_readfirstlane_b32 s43, v3
+; SI-NEXT: v_readfirstlane_b32 s42, v2
+; SI-NEXT: v_readfirstlane_b32 s45, v1
+; SI-NEXT: s_cmp_lg_u32 s44, 0
+; SI-NEXT: v_readfirstlane_b32 s44, v0
+; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s4, 16
-; SI-NEXT: s_and_b32 s4, s50, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 2
-; SI-NEXT: s_lshl_b32 s4, s50, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 1
-; SI-NEXT: s_and_b32 s4, s51, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 4
-; SI-NEXT: s_lshl_b32 s4, s51, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 3
-; SI-NEXT: s_and_b32 s4, s52, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 6
-; SI-NEXT: s_lshl_b32 s4, s52, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 5
-; SI-NEXT: s_and_b32 s4, s53, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 8
-; SI-NEXT: s_lshl_b32 s4, s53, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 7
-; SI-NEXT: s_and_b32 s4, s55, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 10
-; SI-NEXT: s_lshl_b32 s4, s55, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 9
-; SI-NEXT: s_and_b32 s4, s64, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 12
-; SI-NEXT: s_lshl_b32 s4, s64, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 11
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 14
-; SI-NEXT: s_lshl_b32 s4, s29, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 13
-; SI-NEXT: s_and_b32 s4, s28, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 16
-; SI-NEXT: s_lshl_b32 s4, s28, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 15
-; SI-NEXT: s_and_b32 s4, s27, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 18
-; SI-NEXT: s_lshl_b32 s4, s27, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 17
-; SI-NEXT: s_and_b32 s4, s26, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 20
-; SI-NEXT: s_lshl_b32 s4, s26, 16
-; SI-NEXT: s_and_b32 s10, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s30, 16
-; SI-NEXT: s_and_b32 s12, s31, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s31, 16
-; SI-NEXT: s_and_b32 s14, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s34, 16
-; SI-NEXT: s_and_b32 s40, s35, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s35, 16
-; SI-NEXT: s_and_b32 s42, s36, 0xffff0000
-; SI-NEXT: s_lshl_b32 s41, s36, 16
-; SI-NEXT: s_and_b32 s44, s37, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s37, 16
-; SI-NEXT: s_and_b32 s46, s38, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s38, 16
-; SI-NEXT: s_and_b32 s56, s39, 0xffff0000
-; SI-NEXT: s_lshl_b32 s47, s39, 16
-; SI-NEXT: s_and_b32 s58, s48, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s48, 16
-; SI-NEXT: s_and_b32 s60, s49, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s49, 16
-; SI-NEXT: s_and_b32 s78, s54, 0xffff0000
-; SI-NEXT: s_lshl_b32 s77, s54, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 19
-; SI-NEXT: s_and_b32 s62, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s61, s25, 16
-; SI-NEXT: s_and_b32 s72, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s63, s24, 16
-; SI-NEXT: s_and_b32 s74, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s73, s23, 16
-; SI-NEXT: s_and_b32 s76, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s75, s22, 16
-; SI-NEXT: s_and_b32 s88, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s79, s21, 16
-; SI-NEXT: s_and_b32 s90, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s89, s20, 16
-; SI-NEXT: s_and_b32 s92, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s91, s19, 16
-; SI-NEXT: s_and_b32 s94, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s93, s18, 16
-; SI-NEXT: s_and_b32 vcc_lo, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s95, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_hi, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s46, s5, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v33, s46, 0
+; SI-NEXT: s_lshr_b32 s46, s7, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 1
+; SI-NEXT: s_lshr_b32 s46, s9, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 2
+; SI-NEXT: s_lshr_b32 s46, s11, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 3
+; SI-NEXT: s_lshr_b32 s46, s13, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 4
+; SI-NEXT: s_lshr_b32 s46, s15, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 5
+; SI-NEXT: s_lshr_b32 s46, s41, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 6
+; SI-NEXT: s_lshr_b32 s46, s43, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 7
+; SI-NEXT: s_lshr_b32 s46, s45, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 8
+; SI-NEXT: s_lshr_b32 s46, s29, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 9
+; SI-NEXT: s_lshr_b32 s46, s27, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 10
+; SI-NEXT: s_lshr_b32 s46, s25, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 11
+; SI-NEXT: s_lshr_b32 s46, s23, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 12
+; SI-NEXT: s_lshr_b32 s46, s21, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 13
+; SI-NEXT: s_lshr_b32 s46, s19, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 14
+; SI-NEXT: s_lshr_b32 s46, s17, 16
+; SI-NEXT: v_writelane_b32 v33, s46, 15
+; SI-NEXT: s_lshr_b64 s[56:57], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
; SI-NEXT: s_branch .LBB61_3
; SI-NEXT: .LBB61_2:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr91
; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr89
; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr79
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr78
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr73
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr63
; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr47
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; kill: killed $sgpr47
; SI-NEXT: .LBB61_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_mov_b32 s65, s63
-; SI-NEXT: s_mov_b32 s66, s72
-; SI-NEXT: s_mov_b32 s67, s73
-; SI-NEXT: s_mov_b32 s68, s74
-; SI-NEXT: s_mov_b32 s69, s75
-; SI-NEXT: s_mov_b32 s70, s76
-; SI-NEXT: s_mov_b32 s71, s79
-; SI-NEXT: s_mov_b32 s80, s88
-; SI-NEXT: s_mov_b32 s81, s89
-; SI-NEXT: s_mov_b32 s82, s90
-; SI-NEXT: s_mov_b32 s83, s91
-; SI-NEXT: s_mov_b32 s84, s92
-; SI-NEXT: s_mov_b32 s85, s93
-; SI-NEXT: s_mov_b32 s86, s94
-; SI-NEXT: s_mov_b32 s87, s95
-; SI-NEXT: s_mov_b32 s96, vcc_lo
-; SI-NEXT: s_mov_b32 vcc_lo, s61
-; SI-NEXT: s_mov_b32 s97, vcc_hi
-; SI-NEXT: s_mov_b32 vcc_hi, s62
-; SI-NEXT: s_mov_b32 s98, s6
-; SI-NEXT: s_mov_b32 s4, s7
-; SI-NEXT: s_mov_b32 s5, s8
-; SI-NEXT: s_mov_b32 s7, s9
-; SI-NEXT: s_mov_b32 s8, s10
-; SI-NEXT: s_mov_b32 s9, s11
-; SI-NEXT: s_mov_b32 s10, s12
-; SI-NEXT: s_mov_b32 s11, s13
-; SI-NEXT: s_mov_b32 s12, s14
-; SI-NEXT: s_mov_b32 s13, s15
-; SI-NEXT: s_mov_b32 s14, s40
-; SI-NEXT: s_mov_b32 s15, s41
-; SI-NEXT: s_mov_b32 s40, s42
-; SI-NEXT: s_mov_b32 s41, s43
-; SI-NEXT: s_mov_b32 s42, s44
-; SI-NEXT: s_mov_b32 s43, s45
-; SI-NEXT: s_mov_b32 s44, s46
-; SI-NEXT: s_mov_b32 s45, s47
-; SI-NEXT: s_mov_b32 s46, s56
-; SI-NEXT: s_mov_b32 s47, s57
-; SI-NEXT: s_mov_b32 s56, s58
-; SI-NEXT: s_mov_b32 s57, s59
-; SI-NEXT: s_mov_b32 s58, s60
-; SI-NEXT: v_readlane_b32 s59, v34, 1
-; SI-NEXT: v_readlane_b32 s60, v34, 2
-; SI-NEXT: v_readlane_b32 s61, v34, 3
-; SI-NEXT: v_readlane_b32 s62, v34, 4
-; SI-NEXT: v_readlane_b32 s63, v34, 5
-; SI-NEXT: v_readlane_b32 s72, v34, 6
-; SI-NEXT: v_readlane_b32 s73, v34, 7
-; SI-NEXT: v_readlane_b32 s74, v34, 8
-; SI-NEXT: v_readlane_b32 s6, v34, 9
-; SI-NEXT: v_readlane_b32 s75, v34, 10
-; SI-NEXT: v_readlane_b32 s76, v34, 11
-; SI-NEXT: v_readlane_b32 s79, v34, 12
-; SI-NEXT: v_readlane_b32 s88, v34, 13
-; SI-NEXT: v_readlane_b32 s89, v34, 14
-; SI-NEXT: v_readlane_b32 s90, v34, 15
-; SI-NEXT: v_readlane_b32 s91, v34, 16
-; SI-NEXT: v_readlane_b32 s92, v34, 17
-; SI-NEXT: v_readlane_b32 s93, v34, 18
-; SI-NEXT: v_readlane_b32 s94, v34, 19
-; SI-NEXT: v_readlane_b32 s95, v34, 20
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: v_readlane_b32 s36, v33, 0
+; SI-NEXT: v_readlane_b32 s37, v33, 1
+; SI-NEXT: v_readlane_b32 s47, v33, 2
+; SI-NEXT: v_readlane_b32 s57, v33, 3
+; SI-NEXT: v_readlane_b32 s59, v33, 4
+; SI-NEXT: v_readlane_b32 s61, v33, 5
+; SI-NEXT: v_readlane_b32 s38, v33, 6
+; SI-NEXT: v_readlane_b32 s39, v33, 7
+; SI-NEXT: v_readlane_b32 s63, v33, 8
+; SI-NEXT: v_readlane_b32 s48, v33, 9
+; SI-NEXT: v_readlane_b32 s73, v33, 10
+; SI-NEXT: v_readlane_b32 s49, v33, 11
+; SI-NEXT: v_readlane_b32 s75, v33, 12
+; SI-NEXT: v_readlane_b32 s50, v33, 13
+; SI-NEXT: v_readlane_b32 s77, v33, 14
+; SI-NEXT: v_readlane_b32 s51, v33, 15
; SI-NEXT: s_cbranch_scc1 .LBB61_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_u32 s16, s16, 3
-; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: s_add_u32 s18, s18, 3
-; SI-NEXT: s_addc_u32 s19, s19, 0
-; SI-NEXT: s_add_u32 s20, s20, 3
-; SI-NEXT: s_addc_u32 s21, s21, 0
-; SI-NEXT: s_add_u32 s22, s22, 3
-; SI-NEXT: s_addc_u32 s23, s23, 0
-; SI-NEXT: s_add_u32 s24, s24, 3
-; SI-NEXT: s_addc_u32 s25, s25, 0
-; SI-NEXT: s_add_u32 s26, s26, 3
-; SI-NEXT: s_addc_u32 s27, s27, 0
+; SI-NEXT: s_add_u32 s4, s4, 3
+; SI-NEXT: s_addc_u32 s5, s5, 0
+; SI-NEXT: s_add_u32 s6, s6, 3
+; SI-NEXT: s_addc_u32 s7, s7, 0
+; SI-NEXT: s_add_u32 s8, s8, 3
+; SI-NEXT: s_addc_u32 s9, s9, 0
+; SI-NEXT: s_add_u32 s10, s10, 3
+; SI-NEXT: s_addc_u32 s11, s11, 0
+; SI-NEXT: s_add_u32 s12, s12, 3
+; SI-NEXT: s_addc_u32 s13, s13, 0
+; SI-NEXT: s_add_u32 s14, s14, 3
+; SI-NEXT: s_addc_u32 s15, s15, 0
+; SI-NEXT: s_add_u32 s40, s40, 3
+; SI-NEXT: s_addc_u32 s41, s41, 0
+; SI-NEXT: s_add_u32 s42, s42, 3
+; SI-NEXT: s_addc_u32 s43, s43, 0
+; SI-NEXT: s_add_u32 s44, s44, 3
+; SI-NEXT: s_addc_u32 s45, s45, 0
; SI-NEXT: s_add_u32 s28, s28, 3
; SI-NEXT: s_addc_u32 s29, s29, 0
-; SI-NEXT: s_add_u32 s76, s64, 3
-; SI-NEXT: s_addc_u32 s6, s55, 0
-; SI-NEXT: s_add_u32 s75, s54, 3
-; SI-NEXT: s_addc_u32 s73, s53, 0
-; SI-NEXT: s_add_u32 s63, s52, 3
-; SI-NEXT: s_addc_u32 s61, s51, 0
-; SI-NEXT: s_add_u32 s59, s50, 3
-; SI-NEXT: s_addc_u32 s57, s49, 0
-; SI-NEXT: s_add_u32 s47, s48, 3
-; SI-NEXT: s_addc_u32 s45, s39, 0
-; SI-NEXT: s_add_u32 s43, s38, 3
-; SI-NEXT: s_addc_u32 s41, s37, 0
-; SI-NEXT: s_add_u32 s15, s36, 3
-; SI-NEXT: s_addc_u32 s13, s35, 0
-; SI-NEXT: s_add_u32 s11, s34, 3
-; SI-NEXT: s_addc_u32 s9, s31, 0
-; SI-NEXT: s_add_u32 s7, s30, 3
-; SI-NEXT: v_readlane_b32 s4, v34, 0
-; SI-NEXT: s_addc_u32 s4, s4, 0
-; SI-NEXT: s_and_b32 s5, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s8, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s10, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s12, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_and_b32 s14, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_and_b32 s40, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_and_b32 s42, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s41, s41, 16
-; SI-NEXT: s_and_b32 s44, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s43, 16
-; SI-NEXT: s_and_b32 s46, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_and_b32 s56, s47, 0xffff0000
-; SI-NEXT: s_lshl_b32 s47, s47, 16
-; SI-NEXT: s_and_b32 s58, s57, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s57, 16
-; SI-NEXT: s_and_b32 s60, s59, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s59, 16
-; SI-NEXT: s_and_b32 s62, s61, 0xffff0000
-; SI-NEXT: s_lshl_b32 s61, s61, 16
-; SI-NEXT: s_and_b32 s72, s63, 0xffff0000
-; SI-NEXT: s_lshl_b32 s63, s63, 16
-; SI-NEXT: s_and_b32 s74, s73, 0xffff0000
-; SI-NEXT: s_lshl_b32 s73, s73, 16
-; SI-NEXT: s_and_b32 s78, s75, 0xffff0000
-; SI-NEXT: s_lshl_b32 s77, s75, 16
-; SI-NEXT: s_and_b32 s75, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s79, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: s_and_b32 s89, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s29, 16
-; SI-NEXT: s_and_b32 s91, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s28, 16
-; SI-NEXT: s_and_b32 s93, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s27, 16
-; SI-NEXT: s_and_b32 s95, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s26, 16
-; SI-NEXT: s_and_b32 vcc_hi, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, s25, 16
-; SI-NEXT: s_and_b32 s66, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s65, s24, 16
-; SI-NEXT: s_and_b32 s68, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s23, 16
-; SI-NEXT: s_and_b32 s70, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s69, s22, 16
-; SI-NEXT: s_and_b32 s80, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s21, 16
-; SI-NEXT: s_and_b32 s82, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s20, 16
-; SI-NEXT: s_and_b32 s84, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, s19, 16
-; SI-NEXT: s_and_b32 s86, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s18, 16
-; SI-NEXT: s_and_b32 s96, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s17, 16
-; SI-NEXT: s_and_b32 s98, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s97, s16, 16
+; SI-NEXT: s_add_u32 s26, s26, 3
+; SI-NEXT: s_addc_u32 s27, s27, 0
+; SI-NEXT: s_add_u32 s24, s24, 3
+; SI-NEXT: s_addc_u32 s25, s25, 0
+; SI-NEXT: s_add_u32 s22, s22, 3
+; SI-NEXT: s_addc_u32 s23, s23, 0
+; SI-NEXT: s_add_u32 s20, s20, 3
+; SI-NEXT: s_addc_u32 s21, s21, 0
+; SI-NEXT: s_add_u32 s18, s18, 3
+; SI-NEXT: s_addc_u32 s19, s19, 0
+; SI-NEXT: s_add_u32 s16, s16, 3
+; SI-NEXT: s_addc_u32 s17, s17, 0
+; SI-NEXT: s_lshr_b32 s46, s45, 16
+; SI-NEXT: s_lshr_b32 s47, s27, 16
+; SI-NEXT: s_lshr_b32 s76, s23, 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[14:15], 16
+; SI-NEXT: s_lshr_b32 s90, s9, 16
+; SI-NEXT: s_lshr_b32 s91, s11, 16
+; SI-NEXT: s_lshr_b32 s92, s13, 16
+; SI-NEXT: s_lshr_b32 s93, s15, 16
+; SI-NEXT: s_lshr_b32 s78, s19, 16
+; SI-NEXT: s_lshr_b64 s[56:57], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
+; SI-NEXT: s_mov_b32 s63, s46
+; SI-NEXT: s_mov_b32 s73, s47
+; SI-NEXT: s_mov_b32 s75, s76
+; SI-NEXT: s_lshr_b64 s[76:77], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 16
+; SI-NEXT: s_lshr_b32 s36, s5, 16
+; SI-NEXT: s_lshr_b32 s37, s7, 16
+; SI-NEXT: s_lshr_b32 s38, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s43, 16
+; SI-NEXT: s_lshr_b32 s48, s29, 16
+; SI-NEXT: s_lshr_b32 s49, s25, 16
+; SI-NEXT: s_lshr_b32 s50, s21, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
+; SI-NEXT: s_mov_b32 s77, s78
+; SI-NEXT: s_lshr_b64 s[78:79], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
+; SI-NEXT: s_mov_b32 s61, s93
+; SI-NEXT: s_mov_b32 s59, s92
+; SI-NEXT: s_mov_b32 s57, s91
+; SI-NEXT: s_mov_b32 s47, s90
+; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
; SI-NEXT: .LBB61_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s98
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s97
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s96
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s87
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s86
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s85
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s84
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s83
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s82
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s81
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s80
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s71
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s70
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s69
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s68
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s67
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s66
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s65
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, vcc_hi
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, vcc_lo
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s95
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s94
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s93
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s92
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s91
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s90
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s89
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s88
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s79
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s76
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s78
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s77
-; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s74
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s73
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s72
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s63
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s62
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s61
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s60
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s59
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s58
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s57
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s56
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s47
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s46
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s45
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s44
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s43
-; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s42
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s41
-; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s40
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s15
-; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s14
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s13
-; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s12
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s11
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s10
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s5
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
-; SI-NEXT: v_readlane_b32 s30, v33, 33
-; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
-; SI-NEXT: v_readlane_b32 s31, v33, 34
-; SI-NEXT: v_readlane_b32 s98, v33, 32
-; SI-NEXT: v_readlane_b32 s97, v33, 31
-; SI-NEXT: v_readlane_b32 s96, v33, 30
-; SI-NEXT: v_readlane_b32 s87, v33, 29
-; SI-NEXT: v_readlane_b32 s86, v33, 28
-; SI-NEXT: v_readlane_b32 s85, v33, 27
-; SI-NEXT: v_readlane_b32 s84, v33, 26
-; SI-NEXT: v_readlane_b32 s83, v33, 25
-; SI-NEXT: v_readlane_b32 s82, v33, 24
-; SI-NEXT: v_readlane_b32 s81, v33, 23
-; SI-NEXT: v_readlane_b32 s80, v33, 22
-; SI-NEXT: v_readlane_b32 s71, v33, 21
-; SI-NEXT: v_readlane_b32 s70, v33, 20
-; SI-NEXT: v_readlane_b32 s69, v33, 19
-; SI-NEXT: v_readlane_b32 s68, v33, 18
-; SI-NEXT: v_readlane_b32 s67, v33, 17
-; SI-NEXT: v_readlane_b32 s66, v33, 16
-; SI-NEXT: v_readlane_b32 s65, v33, 15
-; SI-NEXT: v_readlane_b32 s64, v33, 14
-; SI-NEXT: v_readlane_b32 s55, v33, 13
-; SI-NEXT: v_readlane_b32 s54, v33, 12
-; SI-NEXT: v_readlane_b32 s53, v33, 11
-; SI-NEXT: v_readlane_b32 s52, v33, 10
-; SI-NEXT: v_readlane_b32 s51, v33, 9
-; SI-NEXT: v_readlane_b32 s50, v33, 8
-; SI-NEXT: v_readlane_b32 s49, v33, 7
-; SI-NEXT: v_readlane_b32 s48, v33, 6
-; SI-NEXT: v_readlane_b32 s39, v33, 5
-; SI-NEXT: v_readlane_b32 s38, v33, 4
-; SI-NEXT: v_readlane_b32 s37, v33, 3
-; SI-NEXT: v_readlane_b32 s36, v33, 2
-; SI-NEXT: v_readlane_b32 s35, v33, 1
-; SI-NEXT: v_readlane_b32 s34, v33, 0
+; SI-NEXT: s_and_b32 s28, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_or_b32 s28, s28, s46
+; SI-NEXT: s_and_b32 s29, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s48, 16
+; SI-NEXT: s_or_b32 s29, s29, s46
+; SI-NEXT: s_and_b32 s44, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s88, 16
+; SI-NEXT: s_or_b32 s44, s44, s46
+; SI-NEXT: s_and_b32 s45, s45, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s63, 16
+; SI-NEXT: s_or_b32 s45, s45, s46
+; SI-NEXT: s_and_b32 s42, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s78, 16
+; SI-NEXT: s_or_b32 s42, s42, s46
+; SI-NEXT: s_and_b32 s43, s43, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s39, 16
+; SI-NEXT: s_or_b32 s43, s43, s46
+; SI-NEXT: s_and_b32 s40, s40, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s76, 16
+; SI-NEXT: s_or_b32 s40, s40, s46
+; SI-NEXT: s_and_b32 s41, s41, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s38, 16
+; SI-NEXT: s_or_b32 s41, s41, s46
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s74, 16
+; SI-NEXT: s_or_b32 s14, s14, s46
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s61, 16
+; SI-NEXT: s_or_b32 s15, s15, s46
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s72, 16
+; SI-NEXT: s_or_b32 s12, s12, s46
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s59, 16
+; SI-NEXT: s_or_b32 s13, s13, s46
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s60, 16
+; SI-NEXT: s_or_b32 s10, s10, s46
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s57, 16
+; SI-NEXT: s_or_b32 s11, s11, s46
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s62, 16
+; SI-NEXT: s_or_b32 s8, s8, s46
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s47, 16
+; SI-NEXT: s_and_b32 s19, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s77, s77, 16
+; SI-NEXT: s_and_b32 s23, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s75, s75, 16
+; SI-NEXT: s_or_b32 s9, s9, s46
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s58, 16
+; SI-NEXT: s_lshl_b32 s79, s34, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s19, s19, s77
+; SI-NEXT: s_lshl_b32 s77, vcc_lo, 16
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s23, s23, s75
+; SI-NEXT: s_lshl_b32 s75, s92, 16
+; SI-NEXT: s_and_b32 s24, s24, 0xffff
+; SI-NEXT: s_or_b32 s6, s6, s46
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s37, 16
+; SI-NEXT: s_or_b32 s16, s16, s79
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s79, s51, 16
+; SI-NEXT: s_or_b32 s20, s20, s77
+; SI-NEXT: s_and_b32 s21, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s77, s50, 16
+; SI-NEXT: s_or_b32 s24, s24, s75
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s75, s49, 16
+; SI-NEXT: s_or_b32 s7, s7, s46
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s56, 16
+; SI-NEXT: s_or_b32 s17, s17, s79
+; SI-NEXT: s_lshl_b32 s79, s30, 16
+; SI-NEXT: s_and_b32 s18, s18, 0xffff
+; SI-NEXT: s_or_b32 s21, s21, s77
+; SI-NEXT: s_lshl_b32 s77, s94, 16
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s25, s25, s75
+; SI-NEXT: s_and_b32 s26, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s75, s90, 16
+; SI-NEXT: s_and_b32 s27, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s73, s73, 16
+; SI-NEXT: s_or_b32 s4, s4, s46
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s36, 16
+; SI-NEXT: s_or_b32 s18, s18, s79
+; SI-NEXT: s_or_b32 s22, s22, s77
+; SI-NEXT: s_or_b32 s26, s26, s75
+; SI-NEXT: s_or_b32 s27, s27, s73
+; SI-NEXT: s_or_b32 s5, s5, s46
+; SI-NEXT: v_readlane_b32 s30, v32, 10
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v14, s44
+; SI-NEXT: v_mov_b32_e32 v15, s45
+; SI-NEXT: v_mov_b32_e32 v16, s42
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v18, s40
+; SI-NEXT: v_mov_b32_e32 v19, s41
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v21, s15
+; SI-NEXT: v_mov_b32_e32 v22, s12
+; SI-NEXT: v_mov_b32_e32 v23, s13
+; SI-NEXT: v_mov_b32_e32 v24, s10
+; SI-NEXT: v_mov_b32_e32 v25, s11
+; SI-NEXT: v_mov_b32_e32 v26, s8
+; SI-NEXT: v_mov_b32_e32 v27, s9
+; SI-NEXT: v_mov_b32_e32 v28, s6
+; SI-NEXT: v_mov_b32_e32 v29, s7
+; SI-NEXT: v_mov_b32_e32 v30, s4
+; SI-NEXT: v_mov_b32_e32 v31, s5
+; SI-NEXT: v_readlane_b32 s31, v32, 11
+; SI-NEXT: v_readlane_b32 s51, v32, 9
+; SI-NEXT: v_readlane_b32 s50, v32, 8
+; SI-NEXT: v_readlane_b32 s49, v32, 7
+; SI-NEXT: v_readlane_b32 s48, v32, 6
+; SI-NEXT: v_readlane_b32 s39, v32, 5
+; SI-NEXT: v_readlane_b32 s38, v32, 4
+; SI-NEXT: v_readlane_b32 s37, v32, 3
+; SI-NEXT: v_readlane_b32 s36, v32, 2
+; SI-NEXT: v_readlane_b32 s35, v32, 1
+; SI-NEXT: v_readlane_b32 s34, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -94092,265 +91653,308 @@ define <16 x i64> @bitcast_v64bf16_to_v16i64(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v62, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v63
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32
+; SI-NEXT: v_mov_b32_e32 v44, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; SI-NEXT: v_mov_b32_e32 v45, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v19
+; SI-NEXT: v_mov_b32_e32 v46, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v20
+; SI-NEXT: v_mov_b32_e32 v47, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
+; SI-NEXT: v_mov_b32_e32 v56, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_mov_b32_e32 v57, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v23
+; SI-NEXT: v_mov_b32_e32 v58, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v24
+; SI-NEXT: v_mov_b32_e32 v59, v3
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v25
+; SI-NEXT: v_mov_b32_e32 v60, v2
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v26
+; SI-NEXT: v_mov_b32_e32 v61, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v27
+; SI-NEXT: v_mov_b32_e32 v42, v12
+; SI-NEXT: v_mov_b32_e32 v43, v11
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v17
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v62
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v42
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v43
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v45
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v46
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v56
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v57
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v58
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v59
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v60
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v61
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v23
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v11
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v60
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v53
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v37
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v38
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v38
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v7
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v63
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v51
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v35
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v34
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB62_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v3, v3, v32
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v62
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v60
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v58
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v56
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v47
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v46
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v45
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v44
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v43
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: v_or_b32_e32 v0, v0, v41
+; SI-NEXT: v_or_b32_e32 v1, v1, v48
+; SI-NEXT: v_or_b32_e32 v2, v2, v40
+; SI-NEXT: v_or_b32_e32 v4, v4, v55
+; SI-NEXT: v_or_b32_e32 v5, v5, v39
+; SI-NEXT: v_or_b32_e32 v6, v6, v54
+; SI-NEXT: v_or_b32_e32 v7, v7, v35
+; SI-NEXT: v_or_b32_e32 v8, v8, v53
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v34
+; SI-NEXT: v_or_b32_e32 v12, v12, v51
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v18, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v13, v13, v37
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v15, v15, v33
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v16, v16, v49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v17, v17, v36
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v63
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
@@ -94384,393 +91988,238 @@ define <16 x i64> @bitcast_v64bf16_to_v16i64(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v43
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v34
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_alignbit_b32 v19, v19, v59, 16
-; SI-NEXT: v_alignbit_b32 v20, v20, v57, 16
-; SI-NEXT: v_alignbit_b32 v21, v21, v46, 16
-; SI-NEXT: v_alignbit_b32 v22, v22, v44, 16
-; SI-NEXT: v_alignbit_b32 v23, v23, v42, 16
-; SI-NEXT: v_alignbit_b32 v24, v24, v40, 16
-; SI-NEXT: v_alignbit_b32 v25, v25, v54, 16
-; SI-NEXT: v_alignbit_b32 v26, v26, v52, 16
-; SI-NEXT: v_alignbit_b32 v27, v27, v50, 16
-; SI-NEXT: v_alignbit_b32 v28, v28, v48, 16
-; SI-NEXT: v_alignbit_b32 v29, v29, v38, 16
-; SI-NEXT: v_alignbit_b32 v30, v30, v35, 16
-; SI-NEXT: v_alignbit_b32 v31, v31, v33, 16
; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: .LBB62_2: ; %Flow
+; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB62_4
+; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v41
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v61
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v48
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_alignbit_b32 v0, v0, v63, 16
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v60
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v59
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v32
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v58
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v55
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v57
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v39
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v54
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v47
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v35
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v46
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v45
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v44
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v43
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v34
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v42
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v51
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v37
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v50
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v62
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v60
-; SI-NEXT: v_alignbit_b32 v18, v18, v61, 16
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: .LBB62_2: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB62_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v49
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v36
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v60
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v58
+; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v47
+; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v45
+; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v43
+; SI-NEXT: s_waitcnt vmcnt(10)
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v41
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v55
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v53
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v51
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v49
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v39
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v36
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v34
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v32
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_alignbit_b32 v15, v16, v15, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_alignbit_b32 v16, v17, v16, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_alignbit_b32 v17, v18, v17, 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v61
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_alignbit_b32 v18, v19, v18, 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v59
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_alignbit_b32 v19, v20, v19, 16
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v57
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_alignbit_b32 v20, v21, v20, 16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v46
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_alignbit_b32 v21, v22, v21, 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v44
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_alignbit_b32 v22, v23, v22, 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v42
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_alignbit_b32 v23, v24, v23, 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_alignbit_b32 v24, v25, v24, 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v54
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_alignbit_b32 v25, v26, v25, 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v52
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_alignbit_b32 v26, v27, v26, 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v50
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_alignbit_b32 v27, v28, v27, 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v48
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_alignbit_b32 v28, v29, v28, 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v38
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_alignbit_b32 v29, v30, v29, 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v35
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_alignbit_b32 v30, v31, v30, 16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v33
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v63
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_alignbit_b32 v31, v32, v31, 16
; SI-NEXT: .LBB62_4: ; %end
@@ -97126,768 +94575,486 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-LABEL: bitcast_v64bf16_to_v16i64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_writelane_b32 v63, s34, 0
-; SI-NEXT: v_writelane_b32 v63, s35, 1
-; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s30, 3
-; SI-NEXT: v_writelane_b32 v63, s31, 4
-; SI-NEXT: v_readfirstlane_b32 s78, v7
-; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s72, v10
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
-; SI-NEXT: v_readfirstlane_b32 s62, v11
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
-; SI-NEXT: v_readfirstlane_b32 s60, v12
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
-; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
-; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s94
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s92, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s92
-; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s90
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v18
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s42
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s78
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v36, s34, 0
+; SI-NEXT: v_writelane_b32 v36, s35, 1
+; SI-NEXT: v_writelane_b32 v36, s36, 2
+; SI-NEXT: v_writelane_b32 v36, s37, 3
+; SI-NEXT: v_writelane_b32 v36, s38, 4
+; SI-NEXT: v_writelane_b32 v36, s39, 5
+; SI-NEXT: v_writelane_b32 v36, s48, 6
+; SI-NEXT: v_writelane_b32 v36, s49, 7
+; SI-NEXT: v_writelane_b32 v36, s50, 8
+; SI-NEXT: v_writelane_b32 v36, s51, 9
+; SI-NEXT: v_writelane_b32 v36, s52, 10
+; SI-NEXT: v_writelane_b32 v36, s53, 11
+; SI-NEXT: v_writelane_b32 v36, s54, 12
+; SI-NEXT: v_writelane_b32 v36, s55, 13
+; SI-NEXT: v_writelane_b32 v36, s64, 14
+; SI-NEXT: v_writelane_b32 v36, s65, 15
+; SI-NEXT: v_writelane_b32 v36, s66, 16
+; SI-NEXT: v_writelane_b32 v36, s67, 17
+; SI-NEXT: v_writelane_b32 v36, s68, 18
+; SI-NEXT: v_writelane_b32 v36, s69, 19
+; SI-NEXT: v_writelane_b32 v36, s70, 20
+; SI-NEXT: v_writelane_b32 v36, s71, 21
+; SI-NEXT: v_writelane_b32 v36, s80, 22
+; SI-NEXT: v_writelane_b32 v36, s81, 23
+; SI-NEXT: v_writelane_b32 v36, s82, 24
+; SI-NEXT: v_writelane_b32 v36, s83, 25
+; SI-NEXT: v_writelane_b32 v36, s84, 26
+; SI-NEXT: v_writelane_b32 v36, s85, 27
+; SI-NEXT: v_writelane_b32 v36, s86, 28
+; SI-NEXT: v_writelane_b32 v36, s87, 29
+; SI-NEXT: v_writelane_b32 v36, s96, 30
+; SI-NEXT: v_writelane_b32 v36, s97, 31
+; SI-NEXT: v_writelane_b32 v36, s98, 32
+; SI-NEXT: v_writelane_b32 v36, s99, 33
+; SI-NEXT: v_writelane_b32 v36, s30, 34
+; SI-NEXT: v_writelane_b32 v36, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
+; SI-NEXT: v_readfirstlane_b32 s11, v15
+; SI-NEXT: v_readfirstlane_b32 s13, v14
+; SI-NEXT: v_readfirstlane_b32 s15, v13
+; SI-NEXT: v_readfirstlane_b32 s73, v12
+; SI-NEXT: v_readfirstlane_b32 s75, v11
+; SI-NEXT: v_readfirstlane_b32 s77, v10
+; SI-NEXT: v_readfirstlane_b32 s79, v9
+; SI-NEXT: v_readfirstlane_b32 s89, v8
+; SI-NEXT: v_readfirstlane_b32 s91, v7
+; SI-NEXT: v_readfirstlane_b32 s93, v6
+; SI-NEXT: v_readfirstlane_b32 s95, v5
+; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s34, v3
+; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: v_readfirstlane_b32 s80, v1
+; SI-NEXT: v_readfirstlane_b32 s83, v0
+; SI-NEXT: s_lshr_b32 vcc_lo, s29, 16
+; SI-NEXT: s_lshr_b32 s30, s28, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s70, s26, 16
+; SI-NEXT: s_lshr_b32 s81, s25, 16
+; SI-NEXT: s_lshr_b32 s84, s24, 16
+; SI-NEXT: s_lshr_b32 s85, s23, 16
+; SI-NEXT: s_lshr_b32 s86, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s87, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s66, s9, 16
+; SI-NEXT: s_lshr_b32 s10, s11, 16
+; SI-NEXT: s_lshr_b32 s12, s13, 16
+; SI-NEXT: s_lshr_b32 s14, s15, 16
+; SI-NEXT: s_lshr_b32 s72, s73, 16
+; SI-NEXT: s_lshr_b32 s74, s75, 16
+; SI-NEXT: s_lshr_b32 s76, s77, 16
+; SI-NEXT: s_lshr_b32 s78, s79, 16
+; SI-NEXT: s_lshr_b32 s88, s89, 16
+; SI-NEXT: s_lshr_b32 s90, s91, 16
+; SI-NEXT: s_lshr_b32 s92, s93, 16
+; SI-NEXT: s_lshr_b32 s94, s95, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s4, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s69, 16
+; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 s82, s83, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: ; implicit-def: $vgpr37 : SGPR spill to VGPR lane
+; SI-NEXT: s_cmp_lg_u32 s5, 0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v37, s67, 0
+; SI-NEXT: v_writelane_b32 v37, s66, 1
; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v37
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v48
-; SI-NEXT: v_mov_b32_e32 v62, v35
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v18
-; SI-NEXT: v_mov_b32_e32 v29, v54
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v30, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s36, s5, s6
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s99, 16
+; SI-NEXT: s_or_b32 s38, s5, s6
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s98, 16
+; SI-NEXT: s_and_b32 s40, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s41, s87, 16
+; SI-NEXT: s_or_b32 s39, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s97, 16
+; SI-NEXT: s_or_b32 s37, s40, s41
+; SI-NEXT: s_or_b32 s40, s5, s6
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s96, 16
+; SI-NEXT: s_or_b32 s41, s5, s6
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s86, 16
+; SI-NEXT: s_or_b32 s42, s5, s6
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s85, 16
+; SI-NEXT: s_or_b32 s43, s5, s6
+; SI-NEXT: s_and_b32 s5, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s84, 16
+; SI-NEXT: s_or_b32 s44, s5, s6
+; SI-NEXT: s_and_b32 s5, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s81, 16
+; SI-NEXT: s_or_b32 s45, s5, s6
+; SI-NEXT: s_and_b32 s5, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s70, 16
+; SI-NEXT: s_or_b32 s46, s5, s6
+; SI-NEXT: s_and_b32 s5, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s35, 16
+; SI-NEXT: s_or_b32 s47, s5, s6
+; SI-NEXT: s_and_b32 s5, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s30, 16
+; SI-NEXT: s_or_b32 s48, s5, s6
+; SI-NEXT: s_and_b32 s5, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s6, vcc_lo, 16
+; SI-NEXT: s_or_b32 s49, s5, s6
+; SI-NEXT: s_and_b32 s5, s83, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s82, 16
+; SI-NEXT: s_or_b32 s50, s5, s6
+; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s71, 16
+; SI-NEXT: s_or_b32 s51, s5, s6
+; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s68, 16
+; SI-NEXT: s_or_b32 s52, s5, s6
+; SI-NEXT: s_and_b32 s5, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s31, 16
+; SI-NEXT: s_or_b32 s53, s5, s6
+; SI-NEXT: s_mov_b32 s6, s4
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_or_b32 s54, s4, s5
+; SI-NEXT: s_and_b32 s4, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_or_b32 s55, s4, s5
+; SI-NEXT: s_and_b32 s4, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_or_b32 s56, s4, s5
+; SI-NEXT: s_and_b32 s4, s91, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_or_b32 s57, s4, s5
+; SI-NEXT: s_and_b32 s4, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_or_b32 s58, s4, s5
+; SI-NEXT: s_and_b32 s4, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: s_or_b32 s59, s4, s5
+; SI-NEXT: s_and_b32 s4, s77, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_or_b32 s60, s4, s5
+; SI-NEXT: s_and_b32 s4, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_or_b32 s61, s4, s5
+; SI-NEXT: s_and_b32 s4, s73, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_or_b32 s62, s4, s5
+; SI-NEXT: s_and_b32 s4, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s63, s4, s5
+; SI-NEXT: s_and_b32 s4, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s64, s4, s5
+; SI-NEXT: s_and_b32 s4, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s10, 16
+; SI-NEXT: s_or_b32 s65, s4, s5
+; SI-NEXT: s_and_b32 s4, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: s_or_b32 s66, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_or_b32 s67, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(2) expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[38:39], 16
-; SI-NEXT: v_mov_b32_e32 v39, v33
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v39
-; SI-NEXT: v_lshr_b64 v[3:4], v[32:33], 16
-; SI-NEXT: v_mov_b32_e32 v4, v41
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[41:42], 16
-; SI-NEXT: v_mov_b32_e32 v5, v26
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[26:27], 16
-; SI-NEXT: v_mov_b32_e32 v6, v25
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v7, v24
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v8
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[24:25], 16
-; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v9, v13
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v33, v61
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v33
-; SI-NEXT: v_lshr_b64 v[10:11], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v13, v60
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[59:60], 16
-; SI-NEXT: v_mov_b32_e32 v14, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v15
-; SI-NEXT: v_mov_b32_e32 v60, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[58:59], 16
-; SI-NEXT: v_mov_b32_e32 v15, v58
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v16
-; SI-NEXT: v_mov_b32_e32 v59, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[57:58], 16
-; SI-NEXT: v_mov_b32_e32 v16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v17
-; SI-NEXT: v_mov_b32_e32 v58, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v17, v56
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v19
-; SI-NEXT: v_mov_b32_e32 v57, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[55:56], 16
-; SI-NEXT: v_mov_b32_e32 v56, v17
-; SI-NEXT: v_lshr_b64 v[16:17], v[51:52], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v20
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v42, v49
-; SI-NEXT: v_mov_b32_e32 v49, v44
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v27
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v54
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v25
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v21
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v22
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v50
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v26
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v24, v53
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[53:54], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v54, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v31
-; SI-NEXT: v_mov_b32_e32 v31, v50
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
-; SI-NEXT: v_lshr_b64 v[25:26], v[40:41], 16
-; SI-NEXT: v_mov_b32_e32 v26, v43
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[26:27], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v27, v28
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v37
-; SI-NEXT: v_mov_b32_e32 v37, v48
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v35
-; SI-NEXT: v_mov_b32_e32 v35, v62
-; SI-NEXT: v_lshr_b64 v[61:62], v[47:48], 16
-; SI-NEXT: v_mov_b32_e32 v44, v49
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[45:46], 16
-; SI-NEXT: v_mov_b32_e32 v46, v30
-; SI-NEXT: v_lshr_b64 v[29:30], v[49:50], 16
-; SI-NEXT: v_mov_b32_e32 v50, v31
-; SI-NEXT: v_lshr_b64 v[30:31], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v61, v33
-; SI-NEXT: v_mov_b32_e32 v33, v39
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v49, v42
; SI-NEXT: s_branch .LBB63_3
; SI-NEXT: .LBB63_2:
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b32 s6, s4
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
; SI-NEXT: .LBB63_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB63_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v31, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v35
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s87, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s99, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s98, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s97, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s96, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s86, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s85, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s84, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s81, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s70, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s35, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_lo, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: s_lshl_b32 s4, s83, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s82, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_lshl_b32 s4, s80, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s71, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: s_lshl_b32 s4, s69, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s68, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: s_lshl_b32 s4, s34, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s31, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: s_lshl_b32 s4, s95, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s94, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: s_lshl_b32 s4, s93, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s92, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_lshl_b32 s4, s91, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s90, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: s_lshl_b32 s4, s89, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s88, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: s_lshl_b32 s4, s79, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s78, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s77, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s76, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s74, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s72, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_add_f32_e32 v32, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 1
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v31, v32
-; SI-NEXT: .LBB63_5: ; %end
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s30, v63, 3
-; SI-NEXT: v_readlane_b32 s31, v63, 4
-; SI-NEXT: v_readlane_b32 s36, v63, 2
-; SI-NEXT: v_readlane_b32 s35, v63, 1
-; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 0
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v31, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_lshr_b64 v[34:35], v[30:31], 16
+; SI-NEXT: v_lshr_b64 v[30:31], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v31, v34
+; SI-NEXT: s_branch .LBB63_6
+; SI-NEXT: .LBB63_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: v_mov_b32_e32 v16, s52
+; SI-NEXT: v_mov_b32_e32 v17, s53
+; SI-NEXT: v_mov_b32_e32 v18, s54
+; SI-NEXT: v_mov_b32_e32 v19, s55
+; SI-NEXT: v_mov_b32_e32 v20, s56
+; SI-NEXT: v_mov_b32_e32 v21, s57
+; SI-NEXT: v_mov_b32_e32 v22, s58
+; SI-NEXT: v_mov_b32_e32 v23, s59
+; SI-NEXT: v_mov_b32_e32 v24, s60
+; SI-NEXT: v_mov_b32_e32 v25, s61
+; SI-NEXT: v_mov_b32_e32 v26, s62
+; SI-NEXT: v_mov_b32_e32 v27, s63
+; SI-NEXT: v_mov_b32_e32 v28, s64
+; SI-NEXT: v_mov_b32_e32 v29, s65
+; SI-NEXT: v_mov_b32_e32 v30, s66
+; SI-NEXT: v_mov_b32_e32 v31, s67
+; SI-NEXT: .LBB63_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v36, 34
+; SI-NEXT: v_readlane_b32 s31, v36, 35
+; SI-NEXT: v_readlane_b32 s99, v36, 33
+; SI-NEXT: v_readlane_b32 s98, v36, 32
+; SI-NEXT: v_readlane_b32 s97, v36, 31
+; SI-NEXT: v_readlane_b32 s96, v36, 30
+; SI-NEXT: v_readlane_b32 s87, v36, 29
+; SI-NEXT: v_readlane_b32 s86, v36, 28
+; SI-NEXT: v_readlane_b32 s85, v36, 27
+; SI-NEXT: v_readlane_b32 s84, v36, 26
+; SI-NEXT: v_readlane_b32 s83, v36, 25
+; SI-NEXT: v_readlane_b32 s82, v36, 24
+; SI-NEXT: v_readlane_b32 s81, v36, 23
+; SI-NEXT: v_readlane_b32 s80, v36, 22
+; SI-NEXT: v_readlane_b32 s71, v36, 21
+; SI-NEXT: v_readlane_b32 s70, v36, 20
+; SI-NEXT: v_readlane_b32 s69, v36, 19
+; SI-NEXT: v_readlane_b32 s68, v36, 18
+; SI-NEXT: v_readlane_b32 s67, v36, 17
+; SI-NEXT: v_readlane_b32 s66, v36, 16
+; SI-NEXT: v_readlane_b32 s65, v36, 15
+; SI-NEXT: v_readlane_b32 s64, v36, 14
+; SI-NEXT: v_readlane_b32 s55, v36, 13
+; SI-NEXT: v_readlane_b32 s54, v36, 12
+; SI-NEXT: v_readlane_b32 s53, v36, 11
+; SI-NEXT: v_readlane_b32 s52, v36, 10
+; SI-NEXT: v_readlane_b32 s51, v36, 9
+; SI-NEXT: v_readlane_b32 s50, v36, 8
+; SI-NEXT: v_readlane_b32 s49, v36, 7
+; SI-NEXT: v_readlane_b32 s48, v36, 6
+; SI-NEXT: v_readlane_b32 s39, v36, 5
+; SI-NEXT: v_readlane_b32 s38, v36, 4
+; SI-NEXT: v_readlane_b32 s37, v36, 3
+; SI-NEXT: v_readlane_b32 s36, v36, 2
+; SI-NEXT: v_readlane_b32 s35, v36, 1
+; SI-NEXT: v_readlane_b32 s34, v36, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64bf16_to_v16i64_scalar:
@@ -125644,435 +122811,204 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr62
; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr60
; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr59
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr58
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr44
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB76_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v52, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v40, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v43, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v46, v3, v2, 16
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_alignbit_b32 v57, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v11
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v9
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v7
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v6
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v5
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v4
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v3
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v2
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v1
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB76_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB76_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
-; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_add_f64 v[22:23], v[22:23], 1.0
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_add_f64 v[20:21], v[20:21], 1.0
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_add_f64 v[18:19], v[18:19], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
+; SI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; SI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
; SI-NEXT: v_add_f64 v[6:7], v[6:7], 1.0
; SI-NEXT: v_add_f64 v[8:9], v[8:9], 1.0
; SI-NEXT: v_add_f64 v[10:11], v[10:11], 1.0
; SI-NEXT: v_add_f64 v[12:13], v[12:13], 1.0
; SI-NEXT: v_add_f64 v[14:15], v[14:15], 1.0
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
-; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
+; SI-NEXT: v_add_f64 v[18:19], v[18:19], 1.0
+; SI-NEXT: v_add_f64 v[20:21], v[20:21], 1.0
+; SI-NEXT: v_add_f64 v[22:23], v[22:23], 1.0
+; SI-NEXT: v_add_f64 v[24:25], v[24:25], 1.0
+; SI-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f64 v[30:31], v[30:31], 1.0
+; SI-NEXT: v_add_f64 v[28:29], v[28:29], 1.0
+; SI-NEXT: v_alignbit_b32 v32, v31, v30, 16
+; SI-NEXT: v_alignbit_b32 v33, v29, v28, 16
+; SI-NEXT: v_alignbit_b32 v34, v27, v26, 16
+; SI-NEXT: v_alignbit_b32 v35, v25, v24, 16
+; SI-NEXT: v_alignbit_b32 v36, v23, v22, 16
+; SI-NEXT: v_alignbit_b32 v37, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v38, v19, v18, 16
+; SI-NEXT: v_alignbit_b32 v39, v17, v16, 16
+; SI-NEXT: v_alignbit_b32 v48, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v49, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v50, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v52, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v40, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v43, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v46, v3, v2, 16
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_alignbit_b32 v57, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v11
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v9
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v7
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v6
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v5
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v4
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v3
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v2
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v1
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v0
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
; SI-NEXT: .LBB76_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v63
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v62
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v60
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v58
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v56
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v46
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v44
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v42
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v40
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; SI-NEXT: v_or_b32_e32 v0, v0, v57
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v63
+; SI-NEXT: v_or_b32_e32 v2, v2, v46
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v62
+; SI-NEXT: v_or_b32_e32 v4, v4, v43
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v61
+; SI-NEXT: v_or_b32_e32 v6, v6, v40
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v60
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; SI-NEXT: v_or_b32_e32 v1, v1, v57
+; SI-NEXT: v_or_b32_e32 v3, v3, v46
+; SI-NEXT: v_or_b32_e32 v5, v5, v43
+; SI-NEXT: v_or_b32_e32 v7, v7, v40
+; SI-NEXT: v_or_b32_e32 v8, v8, v52
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v59
+; SI-NEXT: v_or_b32_e32 v10, v10, v50
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v58
+; SI-NEXT: v_or_b32_e32 v12, v12, v49
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v56
+; SI-NEXT: v_or_b32_e32 v14, v14, v48
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v47
+; SI-NEXT: v_or_b32_e32 v16, v16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v45
+; SI-NEXT: v_or_b32_e32 v18, v18, v38
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v44
+; SI-NEXT: v_or_b32_e32 v20, v20, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v42
+; SI-NEXT: v_or_b32_e32 v22, v22, v36
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v41
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@@ -126089,134 +123025,48 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v24, v24, v35
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v55
+; SI-NEXT: v_or_b32_e32 v26, v26, v34
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v54
+; SI-NEXT: v_or_b32_e32 v28, v28, v33
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v53
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v51
+; SI-NEXT: v_or_b32_e32 v9, v9, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v50
+; SI-NEXT: v_or_b32_e32 v13, v13, v49
+; SI-NEXT: v_or_b32_e32 v15, v15, v48
+; SI-NEXT: v_or_b32_e32 v17, v17, v39
+; SI-NEXT: v_or_b32_e32 v19, v19, v38
+; SI-NEXT: v_or_b32_e32 v21, v21, v37
+; SI-NEXT: v_or_b32_e32 v23, v23, v36
+; SI-NEXT: v_or_b32_e32 v25, v25, v35
+; SI-NEXT: v_or_b32_e32 v27, v27, v34
+; SI-NEXT: v_or_b32_e32 v29, v29, v33
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f64_to_v64bf16:
@@ -126344,23 +123194,23 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: v_writelane_b32 v63, s34, 0
; SI-NEXT: v_writelane_b32 v63, s35, 1
; SI-NEXT: v_writelane_b32 v63, s36, 2
@@ -126376,490 +123226,347 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_writelane_b32 v63, s54, 12
; SI-NEXT: v_writelane_b32 v63, s55, 13
; SI-NEXT: v_writelane_b32 v63, s64, 14
-; SI-NEXT: v_writelane_b32 v63, s65, 15
-; SI-NEXT: v_writelane_b32 v63, s66, 16
-; SI-NEXT: v_writelane_b32 v63, s67, 17
-; SI-NEXT: v_writelane_b32 v63, s68, 18
-; SI-NEXT: v_writelane_b32 v63, s69, 19
-; SI-NEXT: v_writelane_b32 v63, s70, 20
-; SI-NEXT: v_writelane_b32 v63, s71, 21
-; SI-NEXT: v_writelane_b32 v63, s80, 22
-; SI-NEXT: v_writelane_b32 v63, s81, 23
-; SI-NEXT: v_writelane_b32 v63, s82, 24
-; SI-NEXT: v_writelane_b32 v63, s83, 25
-; SI-NEXT: v_writelane_b32 v63, s84, 26
-; SI-NEXT: v_writelane_b32 v63, s85, 27
-; SI-NEXT: v_writelane_b32 v63, s86, 28
-; SI-NEXT: v_writelane_b32 v63, s87, 29
-; SI-NEXT: v_writelane_b32 v63, s96, 30
-; SI-NEXT: v_writelane_b32 v63, s97, 31
-; SI-NEXT: v_writelane_b32 v63, s98, 32
-; SI-NEXT: v_writelane_b32 v63, s99, 33
-; SI-NEXT: v_writelane_b32 v63, s30, 34
-; SI-NEXT: v_writelane_b32 v63, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s44, v18
-; SI-NEXT: v_readfirstlane_b32 s5, v17
-; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_readfirstlane_b32 s7, v15
-; SI-NEXT: v_readfirstlane_b32 s6, v14
-; SI-NEXT: v_readfirstlane_b32 s9, v13
-; SI-NEXT: v_readfirstlane_b32 s8, v12
-; SI-NEXT: v_readfirstlane_b32 s11, v11
-; SI-NEXT: v_readfirstlane_b32 s10, v10
+; SI-NEXT: v_writelane_b32 v63, s30, 15
+; SI-NEXT: v_writelane_b32 v63, s31, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v18
+; SI-NEXT: v_readfirstlane_b32 s45, v17
+; SI-NEXT: v_readfirstlane_b32 s44, v16
+; SI-NEXT: v_readfirstlane_b32 s43, v15
+; SI-NEXT: v_readfirstlane_b32 s42, v14
+; SI-NEXT: v_readfirstlane_b32 s41, v13
+; SI-NEXT: v_readfirstlane_b32 s40, v12
+; SI-NEXT: v_readfirstlane_b32 s15, v11
+; SI-NEXT: v_readfirstlane_b32 s14, v10
; SI-NEXT: v_readfirstlane_b32 s13, v9
; SI-NEXT: v_readfirstlane_b32 s12, v8
-; SI-NEXT: v_readfirstlane_b32 s15, v7
-; SI-NEXT: v_readfirstlane_b32 s14, v6
-; SI-NEXT: v_readfirstlane_b32 s41, v5
-; SI-NEXT: v_readfirstlane_b32 s40, v4
-; SI-NEXT: v_readfirstlane_b32 s43, v3
-; SI-NEXT: v_readfirstlane_b32 s42, v2
-; SI-NEXT: v_readfirstlane_b32 s45, v1
-; SI-NEXT: s_cmp_lg_u32 s44, 0
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
+; SI-NEXT: v_readfirstlane_b32 s11, v7
+; SI-NEXT: v_readfirstlane_b32 s10, v6
+; SI-NEXT: v_readfirstlane_b32 s9, v5
+; SI-NEXT: v_readfirstlane_b32 s8, v4
+; SI-NEXT: v_readfirstlane_b32 s7, v3
+; SI-NEXT: v_readfirstlane_b32 s6, v2
+; SI-NEXT: v_readfirstlane_b32 s5, v1
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s46, s16, 0xffff0000
-; SI-NEXT: v_writelane_b32 v62, s46, 0
-; SI-NEXT: s_lshl_b32 s46, s16, 16
-; SI-NEXT: s_and_b32 s57, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s5, 16
-; SI-NEXT: s_and_b32 s99, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s98, s4, 16
-; SI-NEXT: s_and_b32 s97, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s96, s7, 16
-; SI-NEXT: s_and_b32 s87, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s86, s6, 16
-; SI-NEXT: s_and_b32 s85, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s84, s9, 16
-; SI-NEXT: s_and_b32 s83, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s82, s8, 16
-; SI-NEXT: s_and_b32 s81, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s80, s11, 16
-; SI-NEXT: s_and_b32 s71, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s70, s10, 16
-; SI-NEXT: s_and_b32 s69, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s13, 16
-; SI-NEXT: s_and_b32 s67, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s12, 16
-; SI-NEXT: s_and_b32 s65, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s15, 16
-; SI-NEXT: s_and_b32 s55, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s14, 16
-; SI-NEXT: s_and_b32 s53, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s41, 16
-; SI-NEXT: s_and_b32 s51, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s40, 16
-; SI-NEXT: s_and_b32 s49, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s43, 16
-; SI-NEXT: s_and_b32 s39, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s42, 16
-; SI-NEXT: s_and_b32 s37, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s45, 16
-; SI-NEXT: s_and_b32 s35, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s44, 16
-; SI-NEXT: s_and_b32 s31, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s29, 16
-; SI-NEXT: s_and_b32 vcc_hi, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, s28, 16
-; SI-NEXT: s_and_b32 s95, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s27, 16
-; SI-NEXT: s_and_b32 s93, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s26, 16
-; SI-NEXT: s_and_b32 s91, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s25, 16
-; SI-NEXT: s_and_b32 s89, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s24, 16
-; SI-NEXT: s_and_b32 s79, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s23, 16
-; SI-NEXT: s_and_b32 s77, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s22, 16
-; SI-NEXT: s_and_b32 s75, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s21, 16
-; SI-NEXT: s_and_b32 s73, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s20, 16
-; SI-NEXT: s_and_b32 s63, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s19, 16
-; SI-NEXT: s_and_b32 s61, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s18, 16
-; SI-NEXT: s_and_b32 s59, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s17, 16
-; SI-NEXT: v_writelane_b32 v62, s46, 1
-; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_lshr_b32 s58, s27, 16
+; SI-NEXT: s_lshr_b32 s59, s25, 16
+; SI-NEXT: s_lshr_b32 s56, s17, 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[40:41], 16
+; SI-NEXT: s_lshr_b32 s62, s41, 16
+; SI-NEXT: s_lshr_b32 s63, s23, 16
+; SI-NEXT: s_mov_b32 s47, s56
+; SI-NEXT: s_lshr_b64 s[56:57], s[14:15], 16
+; SI-NEXT: s_mov_b32 s77, s58
+; SI-NEXT: s_mov_b32 s61, s59
+; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
+; SI-NEXT: s_lshr_b32 s64, s45, 16
+; SI-NEXT: s_lshr_b32 s55, s43, 16
+; SI-NEXT: s_lshr_b32 s54, s15, 16
+; SI-NEXT: s_lshr_b32 s53, s13, 16
+; SI-NEXT: s_lshr_b32 s52, s11, 16
+; SI-NEXT: s_lshr_b32 s51, s9, 16
+; SI-NEXT: s_lshr_b32 s50, s7, 16
+; SI-NEXT: s_lshr_b32 s49, s5, 16
+; SI-NEXT: s_lshr_b32 s48, s29, 16
+; SI-NEXT: s_lshr_b32 s39, s21, 16
+; SI-NEXT: s_lshr_b32 s38, s19, 16
+; SI-NEXT: s_mov_b32 s57, s63
+; SI-NEXT: s_mov_b32 s59, s62
+; SI-NEXT: s_lshr_b64 s[62:63], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
; SI-NEXT: s_branch .LBB77_3
; SI-NEXT: .LBB77_2:
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: s_mov_b64 s[36:37], -1
; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr88
; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr78
; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr55
; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: s_mov_b64 s[46:47], -1
; SI-NEXT: .LBB77_3: ; %Flow
-; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
-; SI-NEXT: s_cselect_b32 s46, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s46, 1
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s63, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s63, 1
; SI-NEXT: s_cbranch_scc1 .LBB77_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f64 v[22:23], s[4:5], 1.0
-; SI-NEXT: v_add_f64 v[0:1], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[2:3], s[24:25], 1.0
-; SI-NEXT: v_add_f64 v[14:15], s[14:15], 1.0
-; SI-NEXT: v_add_f64 v[16:17], s[12:13], 1.0
-; SI-NEXT: v_add_f64 v[24:25], s[6:7], 1.0
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_add_f64 v[10:11], s[42:43], 1.0
-; SI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v17
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v15
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v14
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; SI-NEXT: v_add_f64 v[1:2], s[20:21], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
-; SI-NEXT: v_add_f64 v[4:5], s[26:27], 1.0
-; SI-NEXT: v_add_f64 v[12:13], s[40:41], 1.0
-; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f64 v[30:31], s[44:45], 1.0
+; SI-NEXT: v_add_f64 v[28:29], s[42:43], 1.0
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
+; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
+; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v19
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v18
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v11
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v10
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_lshr_b64 v[32:33], v[30:31], 16
+; SI-NEXT: v_add_f64 v[24:25], s[14:15], 1.0
+; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
+; SI-NEXT: v_lshr_b64 v[33:34], v[28:29], 16
+; SI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
+; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
+; SI-NEXT: v_lshr_b64 v[34:35], v[26:27], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[16:17], 16
+; SI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
+; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
+; SI-NEXT: v_lshr_b64 v[35:36], v[24:25], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[14:15], 16
+; SI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
+; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
+; SI-NEXT: v_lshr_b64 v[36:37], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[50:51], v[12:13], 16
+; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v24
-; SI-NEXT: v_add_f64 v[6:7], s[28:29], 1.0
-; SI-NEXT: v_add_f64 v[8:9], s[44:45], 1.0
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v21
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v20
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v13
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v5
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v5
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; SI-NEXT: v_add_f64 v[1:2], s[16:17], 1.0
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v16
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v9
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v8
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v7
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v6
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v0
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_mov_b32_e32 v1, v34
+; SI-NEXT: v_lshr_b64 v[37:38], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[51:52], v[10:11], 16
+; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
+; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; SI-NEXT: v_lshr_b64 v[38:39], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[52:53], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[53:54], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[39:40], v[2:3], 16
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v1
+; SI-NEXT: v_lshr_b64 v[54:55], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[40:41], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
+; SI-NEXT: v_mov_b32_e32 v55, v47
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v19
; SI-NEXT: s_branch .LBB77_6
; SI-NEXT: .LBB77_5:
-; SI-NEXT: v_mov_b32_e32 v0, s56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s99
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s98
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s97
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s96
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s87
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s86
-; SI-NEXT: v_readlane_b32 s4, v62, 0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v32, s64
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v31, s45
+; SI-NEXT: v_mov_b32_e32 v29, s43
+; SI-NEXT: v_mov_b32_e32 v27, s41
+; SI-NEXT: v_mov_b32_e32 v25, s15
+; SI-NEXT: v_mov_b32_e32 v23, s13
+; SI-NEXT: v_mov_b32_e32 v21, s11
+; SI-NEXT: v_mov_b32_e32 v19, s9
+; SI-NEXT: v_mov_b32_e32 v17, s7
+; SI-NEXT: v_mov_b32_e32 v15, s5
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v16, s6
+; SI-NEXT: v_mov_b32_e32 v18, s8
+; SI-NEXT: v_mov_b32_e32 v20, s10
+; SI-NEXT: v_mov_b32_e32 v22, s12
+; SI-NEXT: v_mov_b32_e32 v24, s14
+; SI-NEXT: v_mov_b32_e32 v26, s40
+; SI-NEXT: v_mov_b32_e32 v28, s42
+; SI-NEXT: v_mov_b32_e32 v30, s44
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v42, s55
+; SI-NEXT: v_mov_b32_e32 v43, s59
+; SI-NEXT: v_mov_b32_e32 v44, s54
+; SI-NEXT: v_mov_b32_e32 v45, s53
+; SI-NEXT: v_mov_b32_e32 v46, s52
+; SI-NEXT: v_mov_b32_e32 v41, s51
+; SI-NEXT: v_mov_b32_e32 v47, s50
+; SI-NEXT: v_mov_b32_e32 v56, s49
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_mov_b32_e32 v57, s48
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_mov_b32_e32 v58, s77
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_mov_b32_e32 v59, s61
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_mov_b32_e32 v60, s57
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v61, s39
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v62, s38
+; SI-NEXT: v_mov_b32_e32 v55, s47
+; SI-NEXT: v_mov_b32_e32 v40, s34
+; SI-NEXT: v_mov_b32_e32 v39, s30
+; SI-NEXT: v_mov_b32_e32 v54, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v53, s94
+; SI-NEXT: v_mov_b32_e32 v52, s92
+; SI-NEXT: v_mov_b32_e32 v51, s90
+; SI-NEXT: v_mov_b32_e32 v50, s88
+; SI-NEXT: v_mov_b32_e32 v49, s78
+; SI-NEXT: v_mov_b32_e32 v48, s74
+; SI-NEXT: v_mov_b32_e32 v38, s72
+; SI-NEXT: v_mov_b32_e32 v37, s62
+; SI-NEXT: v_mov_b32_e32 v36, s58
+; SI-NEXT: v_mov_b32_e32 v35, s56
+; SI-NEXT: v_mov_b32_e32 v34, s76
+; SI-NEXT: v_mov_b32_e32 v33, s60
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 1
-; SI-NEXT: v_mov_b32_e32 v1, s57
-; SI-NEXT: v_mov_b32_e32 v51, s85
-; SI-NEXT: v_mov_b32_e32 v50, s84
-; SI-NEXT: v_mov_b32_e32 v53, s83
-; SI-NEXT: v_mov_b32_e32 v52, s82
-; SI-NEXT: v_mov_b32_e32 v55, s81
-; SI-NEXT: v_mov_b32_e32 v54, s80
-; SI-NEXT: v_mov_b32_e32 v41, s71
-; SI-NEXT: v_mov_b32_e32 v40, s70
-; SI-NEXT: v_mov_b32_e32 v43, s69
-; SI-NEXT: v_mov_b32_e32 v42, s68
-; SI-NEXT: v_mov_b32_e32 v45, s67
-; SI-NEXT: v_mov_b32_e32 v44, s66
-; SI-NEXT: v_mov_b32_e32 v47, s65
-; SI-NEXT: v_mov_b32_e32 v46, s64
-; SI-NEXT: v_mov_b32_e32 v57, s55
-; SI-NEXT: v_mov_b32_e32 v56, s54
-; SI-NEXT: v_mov_b32_e32 v59, s53
-; SI-NEXT: v_mov_b32_e32 v58, s52
-; SI-NEXT: v_mov_b32_e32 v61, s51
-; SI-NEXT: v_mov_b32_e32 v60, s50
-; SI-NEXT: v_mov_b32_e32 v38, s49
-; SI-NEXT: v_mov_b32_e32 v37, s48
-; SI-NEXT: v_mov_b32_e32 v33, s39
-; SI-NEXT: v_mov_b32_e32 v32, s38
-; SI-NEXT: v_mov_b32_e32 v31, s37
-; SI-NEXT: v_mov_b32_e32 v30, s36
-; SI-NEXT: v_mov_b32_e32 v29, s35
-; SI-NEXT: v_mov_b32_e32 v28, s34
-; SI-NEXT: v_mov_b32_e32 v27, s31
-; SI-NEXT: v_mov_b32_e32 v26, s30
-; SI-NEXT: v_mov_b32_e32 v25, vcc_hi
-; SI-NEXT: v_mov_b32_e32 v24, vcc_lo
-; SI-NEXT: v_mov_b32_e32 v23, s95
-; SI-NEXT: v_mov_b32_e32 v22, s94
-; SI-NEXT: v_mov_b32_e32 v21, s93
-; SI-NEXT: v_mov_b32_e32 v20, s92
-; SI-NEXT: v_mov_b32_e32 v19, s91
-; SI-NEXT: v_mov_b32_e32 v18, s90
-; SI-NEXT: v_mov_b32_e32 v17, s89
-; SI-NEXT: v_mov_b32_e32 v49, s88
-; SI-NEXT: v_mov_b32_e32 v15, s79
-; SI-NEXT: v_mov_b32_e32 v14, s78
-; SI-NEXT: v_mov_b32_e32 v12, s77
-; SI-NEXT: v_mov_b32_e32 v36, s76
-; SI-NEXT: v_mov_b32_e32 v11, s75
-; SI-NEXT: v_mov_b32_e32 v10, s74
-; SI-NEXT: v_mov_b32_e32 v39, s73
-; SI-NEXT: v_mov_b32_e32 v13, s72
-; SI-NEXT: v_mov_b32_e32 v48, s63
-; SI-NEXT: v_mov_b32_e32 v6, s62
-; SI-NEXT: v_mov_b32_e32 v5, s61
-; SI-NEXT: v_mov_b32_e32 v4, s60
-; SI-NEXT: v_mov_b32_e32 v3, s59
-; SI-NEXT: v_mov_b32_e32 v2, s58
-; SI-NEXT: v_mov_b32_e32 v9, s4
+; SI-NEXT: v_mov_b32_e32 v32, s46
; SI-NEXT: .LBB77_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v48
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v0
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v9
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v33
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v38
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[33:34], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v60
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v46
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v44
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v42
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v50
-; SI-NEXT: v_readlane_b32 s30, v63, 34
-; SI-NEXT: v_readlane_b32 s31, v63, 35
-; SI-NEXT: v_readlane_b32 s99, v63, 33
-; SI-NEXT: v_readlane_b32 s98, v63, 32
-; SI-NEXT: v_readlane_b32 s97, v63, 31
-; SI-NEXT: v_readlane_b32 s96, v63, 30
-; SI-NEXT: v_readlane_b32 s87, v63, 29
-; SI-NEXT: v_readlane_b32 s86, v63, 28
-; SI-NEXT: v_readlane_b32 s85, v63, 27
-; SI-NEXT: v_readlane_b32 s84, v63, 26
-; SI-NEXT: v_readlane_b32 s83, v63, 25
-; SI-NEXT: v_readlane_b32 s82, v63, 24
-; SI-NEXT: v_readlane_b32 s81, v63, 23
-; SI-NEXT: v_readlane_b32 s80, v63, 22
-; SI-NEXT: v_readlane_b32 s71, v63, 21
-; SI-NEXT: v_readlane_b32 s70, v63, 20
-; SI-NEXT: v_readlane_b32 s69, v63, 19
-; SI-NEXT: v_readlane_b32 s68, v63, 18
-; SI-NEXT: v_readlane_b32 s67, v63, 17
-; SI-NEXT: v_readlane_b32 s66, v63, 16
-; SI-NEXT: v_readlane_b32 s65, v63, 15
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v2, v2, v39
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v62
+; SI-NEXT: v_or_b32_e32 v3, v3, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v54
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v4, v4, v39
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v61
+; SI-NEXT: v_or_b32_e32 v5, v5, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v53
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v6, v6, v39
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v60
+; SI-NEXT: v_or_b32_e32 v7, v7, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v52
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v8, v8, v39
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v59
+; SI-NEXT: v_or_b32_e32 v9, v9, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v51
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v10, v10, v39
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v58
+; SI-NEXT: v_or_b32_e32 v11, v11, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v50
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v39
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v57
+; SI-NEXT: v_or_b32_e32 v13, v13, v39
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v49
+; SI-NEXT: v_or_b32_e32 v14, v14, v39
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v56
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v15, v15, v39
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v48
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: v_or_b32_e32 v0, v0, v40
+; SI-NEXT: v_or_b32_e32 v16, v16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v47
+; SI-NEXT: v_or_b32_e32 v18, v18, v38
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v41
+; SI-NEXT: v_or_b32_e32 v20, v20, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v46
+; SI-NEXT: v_or_b32_e32 v22, v22, v36
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v45
+; SI-NEXT: v_or_b32_e32 v24, v24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v44
+; SI-NEXT: v_or_b32_e32 v26, v26, v34
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v43
+; SI-NEXT: v_or_b32_e32 v28, v28, v33
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v42
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: v_and_b32_e32 v32, 0xffff, v31
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_readlane_b32 s30, v63, 15
+; SI-NEXT: v_or_b32_e32 v1, v1, v55
+; SI-NEXT: v_or_b32_e32 v17, v17, v39
+; SI-NEXT: v_or_b32_e32 v19, v19, v38
+; SI-NEXT: v_or_b32_e32 v21, v21, v37
+; SI-NEXT: v_or_b32_e32 v23, v23, v36
+; SI-NEXT: v_or_b32_e32 v25, v25, v35
+; SI-NEXT: v_or_b32_e32 v27, v27, v34
+; SI-NEXT: v_or_b32_e32 v29, v29, v33
+; SI-NEXT: v_readlane_b32 s31, v63, 16
; SI-NEXT: v_readlane_b32 s64, v63, 14
; SI-NEXT: v_readlane_b32 s55, v63, 13
; SI-NEXT: v_readlane_b32 s54, v63, 12
@@ -126875,86 +123582,11 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[4:5], v[5:6], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[24:25], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[26:27], 16
-; SI-NEXT: v_lshr_b64 v[14:15], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[30:31], 16
-; SI-NEXT: v_lshr_b64 v[16:17], v[17:18], 16
-; SI-NEXT: v_lshr_b64 v[17:18], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[60:61], 16
-; SI-NEXT: v_lshr_b64 v[19:20], v[58:59], 16
-; SI-NEXT: v_lshr_b64 v[20:21], v[56:57], 16
-; SI-NEXT: v_lshr_b64 v[21:22], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[42:43], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[40:41], 16
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[25:26], v[54:55], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[48:49], 16
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[29:30], v[38:39], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[30:31], v[36:37], 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[31:32], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v0, v33
-; SI-NEXT: v_mov_b32_e32 v1, v34
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_or_b32_e32 v31, v32, v31
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -127327,265 +123959,308 @@ define <16 x double> @bitcast_v64bf16_to_v16f64(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v63
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v62
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v62, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v53
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v18
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v37
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v38
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32
+; SI-NEXT: v_mov_b32_e32 v44, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; SI-NEXT: v_mov_b32_e32 v45, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v19
+; SI-NEXT: v_mov_b32_e32 v46, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v20
+; SI-NEXT: v_mov_b32_e32 v47, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
+; SI-NEXT: v_mov_b32_e32 v56, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_mov_b32_e32 v57, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v23
+; SI-NEXT: v_mov_b32_e32 v58, v4
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v24
+; SI-NEXT: v_mov_b32_e32 v59, v3
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v25
+; SI-NEXT: v_mov_b32_e32 v60, v2
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v26
+; SI-NEXT: v_mov_b32_e32 v61, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v27
+; SI-NEXT: v_mov_b32_e32 v42, v12
+; SI-NEXT: v_mov_b32_e32 v43, v11
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v17
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v38
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v42
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v43
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v45
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v46
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v56
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v57
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v58
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v59
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v60
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v61
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v23
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v11
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v7
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v63
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v51
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v35
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v34
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB78_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v3, v3, v32
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v62
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v60
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v58
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v56
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v47
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v46
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v45
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v44
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v43
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: v_or_b32_e32 v0, v0, v41
+; SI-NEXT: v_or_b32_e32 v1, v1, v48
+; SI-NEXT: v_or_b32_e32 v2, v2, v40
+; SI-NEXT: v_or_b32_e32 v4, v4, v55
+; SI-NEXT: v_or_b32_e32 v5, v5, v39
+; SI-NEXT: v_or_b32_e32 v6, v6, v54
+; SI-NEXT: v_or_b32_e32 v7, v7, v35
+; SI-NEXT: v_or_b32_e32 v8, v8, v53
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v34
+; SI-NEXT: v_or_b32_e32 v12, v12, v51
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; kill: killed $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v18, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v13, v13, v37
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v15, v15, v33
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v16, v16, v49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v17, v17, v36
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v63
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
@@ -127619,393 +124294,238 @@ define <16 x double> @bitcast_v64bf16_to_v16f64(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v43
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v34
; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: v_alignbit_b32 v19, v19, v59, 16
-; SI-NEXT: v_alignbit_b32 v20, v20, v57, 16
-; SI-NEXT: v_alignbit_b32 v21, v21, v46, 16
-; SI-NEXT: v_alignbit_b32 v22, v22, v44, 16
-; SI-NEXT: v_alignbit_b32 v23, v23, v42, 16
-; SI-NEXT: v_alignbit_b32 v24, v24, v40, 16
-; SI-NEXT: v_alignbit_b32 v25, v25, v54, 16
-; SI-NEXT: v_alignbit_b32 v26, v26, v52, 16
-; SI-NEXT: v_alignbit_b32 v27, v27, v50, 16
-; SI-NEXT: v_alignbit_b32 v28, v28, v48, 16
-; SI-NEXT: v_alignbit_b32 v29, v29, v38, 16
-; SI-NEXT: v_alignbit_b32 v30, v30, v35, 16
-; SI-NEXT: v_alignbit_b32 v31, v31, v33, 16
; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr63
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: .LBB78_2: ; %Flow
+; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB78_4
+; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v62
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v41
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v61
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v48
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_alignbit_b32 v0, v0, v63, 16
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v60
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v59
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v32
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v58
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v55
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v57
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v39
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v54
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v47
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v35
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v46
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v45
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v44
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v43
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v34
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v42
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v51
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v37
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v50
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v62
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v60
-; SI-NEXT: v_alignbit_b32 v18, v18, v61, 16
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: .LBB78_2: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB78_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v49
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v36
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v60
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v58
+; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v47
+; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v45
+; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v43
+; SI-NEXT: s_waitcnt vmcnt(10)
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v41
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v55
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v53
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v51
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v49
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v39
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v36
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v34
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v32
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_alignbit_b32 v15, v16, v15, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_alignbit_b32 v16, v17, v16, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_alignbit_b32 v17, v18, v17, 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v61
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_alignbit_b32 v18, v19, v18, 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v59
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_alignbit_b32 v19, v20, v19, 16
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v57
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_alignbit_b32 v20, v21, v20, 16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v46
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_alignbit_b32 v21, v22, v21, 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v44
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_alignbit_b32 v22, v23, v22, 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v42
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_alignbit_b32 v23, v24, v23, 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_alignbit_b32 v24, v25, v24, 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v54
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_alignbit_b32 v25, v26, v25, 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v52
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_alignbit_b32 v26, v27, v26, 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v50
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_alignbit_b32 v27, v28, v27, 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v48
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_alignbit_b32 v28, v29, v28, 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v38
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_alignbit_b32 v29, v30, v29, 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v35
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_alignbit_b32 v30, v31, v30, 16
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v33
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v63
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_alignbit_b32 v31, v32, v31, 16
; SI-NEXT: .LBB78_4: ; %end
@@ -130361,768 +126881,486 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-LABEL: bitcast_v64bf16_to_v16f64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_writelane_b32 v63, s34, 0
-; SI-NEXT: v_writelane_b32 v63, s35, 1
-; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s30, 3
-; SI-NEXT: v_writelane_b32 v63, s31, 4
-; SI-NEXT: v_readfirstlane_b32 s78, v7
-; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s72, v10
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
-; SI-NEXT: v_readfirstlane_b32 s62, v11
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
-; SI-NEXT: v_readfirstlane_b32 s60, v12
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
-; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
-; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s94
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s92, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s92
-; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s90
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v18
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s42
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s78
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v36, s34, 0
+; SI-NEXT: v_writelane_b32 v36, s35, 1
+; SI-NEXT: v_writelane_b32 v36, s36, 2
+; SI-NEXT: v_writelane_b32 v36, s37, 3
+; SI-NEXT: v_writelane_b32 v36, s38, 4
+; SI-NEXT: v_writelane_b32 v36, s39, 5
+; SI-NEXT: v_writelane_b32 v36, s48, 6
+; SI-NEXT: v_writelane_b32 v36, s49, 7
+; SI-NEXT: v_writelane_b32 v36, s50, 8
+; SI-NEXT: v_writelane_b32 v36, s51, 9
+; SI-NEXT: v_writelane_b32 v36, s52, 10
+; SI-NEXT: v_writelane_b32 v36, s53, 11
+; SI-NEXT: v_writelane_b32 v36, s54, 12
+; SI-NEXT: v_writelane_b32 v36, s55, 13
+; SI-NEXT: v_writelane_b32 v36, s64, 14
+; SI-NEXT: v_writelane_b32 v36, s65, 15
+; SI-NEXT: v_writelane_b32 v36, s66, 16
+; SI-NEXT: v_writelane_b32 v36, s67, 17
+; SI-NEXT: v_writelane_b32 v36, s68, 18
+; SI-NEXT: v_writelane_b32 v36, s69, 19
+; SI-NEXT: v_writelane_b32 v36, s70, 20
+; SI-NEXT: v_writelane_b32 v36, s71, 21
+; SI-NEXT: v_writelane_b32 v36, s80, 22
+; SI-NEXT: v_writelane_b32 v36, s81, 23
+; SI-NEXT: v_writelane_b32 v36, s82, 24
+; SI-NEXT: v_writelane_b32 v36, s83, 25
+; SI-NEXT: v_writelane_b32 v36, s84, 26
+; SI-NEXT: v_writelane_b32 v36, s85, 27
+; SI-NEXT: v_writelane_b32 v36, s86, 28
+; SI-NEXT: v_writelane_b32 v36, s87, 29
+; SI-NEXT: v_writelane_b32 v36, s96, 30
+; SI-NEXT: v_writelane_b32 v36, s97, 31
+; SI-NEXT: v_writelane_b32 v36, s98, 32
+; SI-NEXT: v_writelane_b32 v36, s99, 33
+; SI-NEXT: v_writelane_b32 v36, s30, 34
+; SI-NEXT: v_writelane_b32 v36, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
+; SI-NEXT: v_readfirstlane_b32 s11, v15
+; SI-NEXT: v_readfirstlane_b32 s13, v14
+; SI-NEXT: v_readfirstlane_b32 s15, v13
+; SI-NEXT: v_readfirstlane_b32 s73, v12
+; SI-NEXT: v_readfirstlane_b32 s75, v11
+; SI-NEXT: v_readfirstlane_b32 s77, v10
+; SI-NEXT: v_readfirstlane_b32 s79, v9
+; SI-NEXT: v_readfirstlane_b32 s89, v8
+; SI-NEXT: v_readfirstlane_b32 s91, v7
+; SI-NEXT: v_readfirstlane_b32 s93, v6
+; SI-NEXT: v_readfirstlane_b32 s95, v5
+; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s34, v3
+; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: v_readfirstlane_b32 s80, v1
+; SI-NEXT: v_readfirstlane_b32 s83, v0
+; SI-NEXT: s_lshr_b32 vcc_lo, s29, 16
+; SI-NEXT: s_lshr_b32 s30, s28, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s70, s26, 16
+; SI-NEXT: s_lshr_b32 s81, s25, 16
+; SI-NEXT: s_lshr_b32 s84, s24, 16
+; SI-NEXT: s_lshr_b32 s85, s23, 16
+; SI-NEXT: s_lshr_b32 s86, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s87, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s66, s9, 16
+; SI-NEXT: s_lshr_b32 s10, s11, 16
+; SI-NEXT: s_lshr_b32 s12, s13, 16
+; SI-NEXT: s_lshr_b32 s14, s15, 16
+; SI-NEXT: s_lshr_b32 s72, s73, 16
+; SI-NEXT: s_lshr_b32 s74, s75, 16
+; SI-NEXT: s_lshr_b32 s76, s77, 16
+; SI-NEXT: s_lshr_b32 s78, s79, 16
+; SI-NEXT: s_lshr_b32 s88, s89, 16
+; SI-NEXT: s_lshr_b32 s90, s91, 16
+; SI-NEXT: s_lshr_b32 s92, s93, 16
+; SI-NEXT: s_lshr_b32 s94, s95, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s4, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s69, 16
+; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 s82, s83, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: ; implicit-def: $vgpr37 : SGPR spill to VGPR lane
+; SI-NEXT: s_cmp_lg_u32 s5, 0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v37, s67, 0
+; SI-NEXT: v_writelane_b32 v37, s66, 1
; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v37
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v48
-; SI-NEXT: v_mov_b32_e32 v62, v35
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v18
-; SI-NEXT: v_mov_b32_e32 v29, v54
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v30, v46
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s36, s5, s6
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s99, 16
+; SI-NEXT: s_or_b32 s38, s5, s6
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s98, 16
+; SI-NEXT: s_and_b32 s40, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s41, s87, 16
+; SI-NEXT: s_or_b32 s39, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s97, 16
+; SI-NEXT: s_or_b32 s37, s40, s41
+; SI-NEXT: s_or_b32 s40, s5, s6
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s96, 16
+; SI-NEXT: s_or_b32 s41, s5, s6
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s86, 16
+; SI-NEXT: s_or_b32 s42, s5, s6
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s85, 16
+; SI-NEXT: s_or_b32 s43, s5, s6
+; SI-NEXT: s_and_b32 s5, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s84, 16
+; SI-NEXT: s_or_b32 s44, s5, s6
+; SI-NEXT: s_and_b32 s5, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s81, 16
+; SI-NEXT: s_or_b32 s45, s5, s6
+; SI-NEXT: s_and_b32 s5, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s70, 16
+; SI-NEXT: s_or_b32 s46, s5, s6
+; SI-NEXT: s_and_b32 s5, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s35, 16
+; SI-NEXT: s_or_b32 s47, s5, s6
+; SI-NEXT: s_and_b32 s5, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s30, 16
+; SI-NEXT: s_or_b32 s48, s5, s6
+; SI-NEXT: s_and_b32 s5, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s6, vcc_lo, 16
+; SI-NEXT: s_or_b32 s49, s5, s6
+; SI-NEXT: s_and_b32 s5, s83, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s82, 16
+; SI-NEXT: s_or_b32 s50, s5, s6
+; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s71, 16
+; SI-NEXT: s_or_b32 s51, s5, s6
+; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s68, 16
+; SI-NEXT: s_or_b32 s52, s5, s6
+; SI-NEXT: s_and_b32 s5, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s31, 16
+; SI-NEXT: s_or_b32 s53, s5, s6
+; SI-NEXT: s_mov_b32 s6, s4
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_or_b32 s54, s4, s5
+; SI-NEXT: s_and_b32 s4, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_or_b32 s55, s4, s5
+; SI-NEXT: s_and_b32 s4, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_or_b32 s56, s4, s5
+; SI-NEXT: s_and_b32 s4, s91, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_or_b32 s57, s4, s5
+; SI-NEXT: s_and_b32 s4, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_or_b32 s58, s4, s5
+; SI-NEXT: s_and_b32 s4, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: s_or_b32 s59, s4, s5
+; SI-NEXT: s_and_b32 s4, s77, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_or_b32 s60, s4, s5
+; SI-NEXT: s_and_b32 s4, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_or_b32 s61, s4, s5
+; SI-NEXT: s_and_b32 s4, s73, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_or_b32 s62, s4, s5
+; SI-NEXT: s_and_b32 s4, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s63, s4, s5
+; SI-NEXT: s_and_b32 s4, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s64, s4, s5
+; SI-NEXT: s_and_b32 s4, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s10, 16
+; SI-NEXT: s_or_b32 s65, s4, s5
+; SI-NEXT: s_and_b32 s4, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: s_or_b32 s66, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_or_b32 s67, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(2) expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[38:39], 16
-; SI-NEXT: v_mov_b32_e32 v39, v33
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v39
-; SI-NEXT: v_lshr_b64 v[3:4], v[32:33], 16
-; SI-NEXT: v_mov_b32_e32 v4, v41
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[41:42], 16
-; SI-NEXT: v_mov_b32_e32 v5, v26
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[26:27], 16
-; SI-NEXT: v_mov_b32_e32 v6, v25
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v7, v24
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v8
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[24:25], 16
-; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v9, v13
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v33, v61
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v33
-; SI-NEXT: v_lshr_b64 v[10:11], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v13, v60
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[59:60], 16
-; SI-NEXT: v_mov_b32_e32 v14, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v15
-; SI-NEXT: v_mov_b32_e32 v60, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[58:59], 16
-; SI-NEXT: v_mov_b32_e32 v15, v58
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v16
-; SI-NEXT: v_mov_b32_e32 v59, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[57:58], 16
-; SI-NEXT: v_mov_b32_e32 v16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v17
-; SI-NEXT: v_mov_b32_e32 v58, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v17, v56
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v19
-; SI-NEXT: v_mov_b32_e32 v57, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[55:56], 16
-; SI-NEXT: v_mov_b32_e32 v56, v17
-; SI-NEXT: v_lshr_b64 v[16:17], v[51:52], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v20
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v42, v49
-; SI-NEXT: v_mov_b32_e32 v49, v44
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v27
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v54
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v25
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v21
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v22
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v50
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v26
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v24, v53
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[53:54], 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v54, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v31
-; SI-NEXT: v_mov_b32_e32 v31, v50
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
-; SI-NEXT: v_lshr_b64 v[25:26], v[40:41], 16
-; SI-NEXT: v_mov_b32_e32 v26, v43
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[26:27], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v27, v28
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v37
-; SI-NEXT: v_mov_b32_e32 v37, v48
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v35
-; SI-NEXT: v_mov_b32_e32 v35, v62
-; SI-NEXT: v_lshr_b64 v[61:62], v[47:48], 16
-; SI-NEXT: v_mov_b32_e32 v44, v49
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[27:28], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[45:46], 16
-; SI-NEXT: v_mov_b32_e32 v46, v30
-; SI-NEXT: v_lshr_b64 v[29:30], v[49:50], 16
-; SI-NEXT: v_mov_b32_e32 v50, v31
-; SI-NEXT: v_lshr_b64 v[30:31], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v61, v33
-; SI-NEXT: v_mov_b32_e32 v33, v39
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v49, v42
; SI-NEXT: s_branch .LBB79_3
; SI-NEXT: .LBB79_2:
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v52, v40
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b32 s6, s4
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
; SI-NEXT: .LBB79_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB79_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v31, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v35
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s87, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s99, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s98, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s97, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s96, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s86, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s85, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s84, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s81, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s70, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s35, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_lo, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: s_lshl_b32 s4, s83, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s82, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_lshl_b32 s4, s80, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s71, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: s_lshl_b32 s4, s69, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s68, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: s_lshl_b32 s4, s34, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s31, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v31
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: s_lshl_b32 s4, s95, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s94, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: s_lshl_b32 s4, s93, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s92, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_lshl_b32 s4, s91, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s90, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: s_lshl_b32 s4, s89, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s88, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: s_lshl_b32 s4, s79, 16
+; SI-NEXT: v_add_f32_e32 v23, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s78, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s77, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s76, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s74, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s72, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v27, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_add_f32_e32 v32, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 1
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v31, v32
-; SI-NEXT: .LBB79_5: ; %end
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s30, v63, 3
-; SI-NEXT: v_readlane_b32 s31, v63, 4
-; SI-NEXT: v_readlane_b32 s36, v63, 2
-; SI-NEXT: v_readlane_b32 s35, v63, 1
-; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; SI-NEXT: v_add_f32_e32 v30, s4, v31
+; SI-NEXT: v_readlane_b32 s4, v37, 0
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v31, s4, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_lshr_b64 v[34:35], v[30:31], 16
+; SI-NEXT: v_lshr_b64 v[30:31], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v31, v34
+; SI-NEXT: s_branch .LBB79_6
+; SI-NEXT: .LBB79_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: v_mov_b32_e32 v16, s52
+; SI-NEXT: v_mov_b32_e32 v17, s53
+; SI-NEXT: v_mov_b32_e32 v18, s54
+; SI-NEXT: v_mov_b32_e32 v19, s55
+; SI-NEXT: v_mov_b32_e32 v20, s56
+; SI-NEXT: v_mov_b32_e32 v21, s57
+; SI-NEXT: v_mov_b32_e32 v22, s58
+; SI-NEXT: v_mov_b32_e32 v23, s59
+; SI-NEXT: v_mov_b32_e32 v24, s60
+; SI-NEXT: v_mov_b32_e32 v25, s61
+; SI-NEXT: v_mov_b32_e32 v26, s62
+; SI-NEXT: v_mov_b32_e32 v27, s63
+; SI-NEXT: v_mov_b32_e32 v28, s64
+; SI-NEXT: v_mov_b32_e32 v29, s65
+; SI-NEXT: v_mov_b32_e32 v30, s66
+; SI-NEXT: v_mov_b32_e32 v31, s67
+; SI-NEXT: .LBB79_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v36, 34
+; SI-NEXT: v_readlane_b32 s31, v36, 35
+; SI-NEXT: v_readlane_b32 s99, v36, 33
+; SI-NEXT: v_readlane_b32 s98, v36, 32
+; SI-NEXT: v_readlane_b32 s97, v36, 31
+; SI-NEXT: v_readlane_b32 s96, v36, 30
+; SI-NEXT: v_readlane_b32 s87, v36, 29
+; SI-NEXT: v_readlane_b32 s86, v36, 28
+; SI-NEXT: v_readlane_b32 s85, v36, 27
+; SI-NEXT: v_readlane_b32 s84, v36, 26
+; SI-NEXT: v_readlane_b32 s83, v36, 25
+; SI-NEXT: v_readlane_b32 s82, v36, 24
+; SI-NEXT: v_readlane_b32 s81, v36, 23
+; SI-NEXT: v_readlane_b32 s80, v36, 22
+; SI-NEXT: v_readlane_b32 s71, v36, 21
+; SI-NEXT: v_readlane_b32 s70, v36, 20
+; SI-NEXT: v_readlane_b32 s69, v36, 19
+; SI-NEXT: v_readlane_b32 s68, v36, 18
+; SI-NEXT: v_readlane_b32 s67, v36, 17
+; SI-NEXT: v_readlane_b32 s66, v36, 16
+; SI-NEXT: v_readlane_b32 s65, v36, 15
+; SI-NEXT: v_readlane_b32 s64, v36, 14
+; SI-NEXT: v_readlane_b32 s55, v36, 13
+; SI-NEXT: v_readlane_b32 s54, v36, 12
+; SI-NEXT: v_readlane_b32 s53, v36, 11
+; SI-NEXT: v_readlane_b32 s52, v36, 10
+; SI-NEXT: v_readlane_b32 s51, v36, 9
+; SI-NEXT: v_readlane_b32 s50, v36, 8
+; SI-NEXT: v_readlane_b32 s49, v36, 7
+; SI-NEXT: v_readlane_b32 s48, v36, 6
+; SI-NEXT: v_readlane_b32 s39, v36, 5
+; SI-NEXT: v_readlane_b32 s38, v36, 4
+; SI-NEXT: v_readlane_b32 s37, v36, 3
+; SI-NEXT: v_readlane_b32 s36, v36, 2
+; SI-NEXT: v_readlane_b32 s35, v36, 1
+; SI-NEXT: v_readlane_b32 s34, v36, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64bf16_to_v16f64_scalar:
@@ -140317,1996 +136555,1999 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v59, v24
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v47, v16
+; SI-NEXT: v_mov_b32_e32 v61, v14
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v60, v6
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:388
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:388
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:48
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:56
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:64
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:80
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:88
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:112
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:120
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:128
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:152
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:176
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:108
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:88
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:96
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:80
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:128
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:112
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:152
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:184
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92
+; SI-NEXT: v_lshlrev_b32_e32 v40, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v9
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v5
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v17
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v7
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v25
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v11
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:4
+; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v7
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v13
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v15
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v19
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v11
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v21
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v23
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v27
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v19
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v29
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v27
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v43, v8
; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr49
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v17
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v26
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v10
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v14
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v12
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v20
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v22
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v18
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v24
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v18
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:116
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:136
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:28
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v6
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:100
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v5
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v28
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v30
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; kill: killed $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:172
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:168
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:176
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:168
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v10
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v32
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v33
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v34
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:208
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:196
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v11
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v18, 8, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:208
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v1
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v0
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:280
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v4
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:296
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:292
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v5
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v43, 8, v11
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:320
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:336
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:344
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:316
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:332
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:328
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:324
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:340
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v45, 24, v12
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v3
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:232
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v59, 8, v14
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:352
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:368
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:376
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:348
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:364
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:360
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:356
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:372
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v60, 24, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:380
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshlrev_b32_e32 v61, 24, v13
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; kill: killed $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v63, 8, v14
-; SI-NEXT: ; kill: killed $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; kill: killed $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v62, 24, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:28
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v12
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268
+; SI-NEXT: v_lshlrev_b32_e32 v63, 8, v2
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:264
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:12
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v0
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:100
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:8
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:300
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v13
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:92
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v15
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:328
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116
+; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v14
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v16
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:84
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v2
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:376
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v0
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:384
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:364
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:360
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:356
+; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v18
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v15
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v62, 24, v19
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v20
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:72
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:68
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:20
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:44
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:44
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; kill: killed $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB88_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v31, v7, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v20
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v6
-; SI-NEXT: v_or_b32_e32 v32, v4, v2
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v29, v45, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: v_or_b32_e32 v33, v0, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v51, 0xff, v47
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v48, 0xff, v43
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v34, v47, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v41
-; SI-NEXT: v_or_b32_e32 v0, v0, v59
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v8
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v27, v54, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v9
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v2, v2, v43
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_or_b32_e32 v0, v57, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_or_b32_e32 v2, v2, v58
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v3, v62, v3
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
-; SI-NEXT: v_or_b32_e32 v12, v13, v12
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v1, v5, v1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v42
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v35, v60, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v56
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v46
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v36, v5, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v44
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v37, v61, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v58
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v57
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v4, v4, v63
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v60
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_or_b32_e32 v38, v62, v5
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v4
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_or_b32_e32 v12, v13, v12
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
+; SI-NEXT: v_mov_b32_e32 v60, v32
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v12, v12, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v5, v1, v5
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v9, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 24, v14
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_or_b32_e32 v9, v1, v9
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v11, v10, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v61
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_or_b32_e32 v10, v1, v10
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v15
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
+; SI-NEXT: v_or_b32_e32 v12, v1, v12
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v14, v14, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v14
+; SI-NEXT: v_and_b32_e32 v13, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v13, v13, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v14, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v17, v1, v14
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v21, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v21
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v14, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v14, v1, v14
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v15, v15, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v39, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v16, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v25, v1, v16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v16, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v16, v1, v16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v48, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v19, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v19, v19, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v20, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v49, v1, v20
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v49, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v49
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v20, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v20, v1, v20
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v21, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v21, v21, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v50, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v50, v1, v22
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v22, v1, v22
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v51, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v23, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v23, v23, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v24, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v52, v1, v24
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v52, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v52
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v24, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v24, v1, v24
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v27, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v27, v27, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v53, v15, v19
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v28, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v22, v19, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_or_b32_e32 v53, v1, v28
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v19
-; SI-NEXT: v_or_b32_e32 v26, v19, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v48
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; SI-NEXT: v_and_b32_e32 v28, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v25, v3, v15
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v28, v1, v28
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v29, v29, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v54, v1, v30
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v30, v1, v30
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v33, v31, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v55, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v34, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v35, v31, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v41, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v36, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v37, v31, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v42, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v38, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v39, v31, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v44, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v45, v1, v31
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v63, v31, v63
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_or_b32_e32 v26, v26, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_or_b32_e32 v7, v7, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v32, v31, v4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v4, v46, v4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_or_b32_e32 v46, v56, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v31, v31, v40
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v59
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_or_b32_e32 v31, v31, v9
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_or_b32_e32 v48, v48, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v51, v51, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v56, v40, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v57, v40, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v58, v40, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v62, v40, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v18, v40, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v6, v40, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v40, v1
+; SI-NEXT: v_and_b32_e32 v40, 0xff, v60
+; SI-NEXT: v_or_b32_e32 v40, v40, v43
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v43, 0xff, v43
+; SI-NEXT: v_or_b32_e32 v60, v43, v47
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_and_b32_e32 v43, 0xff, v43
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v24, v17, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v3, v3, v18
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v3
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: v_or_b32_e32 v47, v43, v47
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v43, 0xff, v43
+; SI-NEXT: v_or_b32_e32 v61, v43, v59
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_and_b32_e32 v43, 0xff, v43
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v17
-; SI-NEXT: v_or_b32_e32 v30, v17, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: v_or_b32_e32 v43, v43, v59
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v28, v10, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v39
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v50
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v53
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: v_and_b32_e32 v59, 0xff, v59
+; SI-NEXT: v_or_b32_e32 v59, v59, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v8, v8, v5
+; SI-NEXT: v_alignbit_b32 v9, v8, v9, 16
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v11
+; SI-NEXT: v_or_b32_e32 v11, v9, v10
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v48
+; SI-NEXT: v_or_b32_e32 v48, v9, v12
+; SI-NEXT: v_alignbit_b32 v9, v11, v12, 16
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v12, v9, v17
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v51
+; SI-NEXT: v_or_b32_e32 v9, v9, v14
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v9, v12, v14, 16
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v51, v9, v25
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v56
+; SI-NEXT: v_or_b32_e32 v15, v9, v16
+; SI-NEXT: v_alignbit_b32 v9, v51, v16, 16
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v19
+; SI-NEXT: v_or_b32_e32 v9, v9, v49
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v57
+; SI-NEXT: v_or_b32_e32 v16, v13, v20
+; SI-NEXT: v_alignbit_b32 v13, v9, v20, 16
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v31, v13, v50
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v58
+; SI-NEXT: v_or_b32_e32 v13, v13, v22
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v13, v31, v22, 16
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v62
+; SI-NEXT: v_or_b32_e32 v13, v13, v52
+; SI-NEXT: v_or_b32_e32 v14, v14, v24
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v14, v13, v24, 16
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v24, v14, v53
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v14, v28
+; SI-NEXT: v_alignbit_b32 v14, v24, v28, 16
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v21, v14, v54
+; SI-NEXT: v_or_b32_e32 v22, v6, v30
+; SI-NEXT: v_alignbit_b32 v6, v21, v30, 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v33
+; SI-NEXT: v_or_b32_e32 v19, v6, v55
+; SI-NEXT: v_or_b32_e32 v29, v1, v34
+; SI-NEXT: v_alignbit_b32 v1, v19, v34, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v35
+; SI-NEXT: v_or_b32_e32 v23, v1, v41
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v40
+; SI-NEXT: v_or_b32_e32 v20, v1, v36
+; SI-NEXT: v_alignbit_b32 v1, v23, v36, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v37
+; SI-NEXT: v_or_b32_e32 v30, v1, v42
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v60
+; SI-NEXT: v_or_b32_e32 v27, v1, v38
+; SI-NEXT: v_alignbit_b32 v1, v30, v38, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v39
+; SI-NEXT: v_or_b32_e32 v28, v1, v44
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v47
+; SI-NEXT: v_or_b32_e32 v34, v1, v45
+; SI-NEXT: v_alignbit_b32 v1, v28, v45, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v63
+; SI-NEXT: v_or_b32_e32 v33, v1, v26
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: v_or_b32_e32 v36, v1, v7
+; SI-NEXT: v_alignbit_b32 v1, v33, v7, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v32
+; SI-NEXT: v_or_b32_e32 v35, v1, v4
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v43
+; SI-NEXT: v_or_b32_e32 v38, v1, v46
+; SI-NEXT: v_alignbit_b32 v1, v35, v46, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v37, v1, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v59
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_or_b32_e32 v39, v1, v3
+; SI-NEXT: v_alignbit_b32 v1, v37, v3, 16
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v49
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v50
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v52
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v53
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v54
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v55
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v41
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v42
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v4
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: .LBB88_2: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
+; SI-NEXT: s_xor_b64 exec, exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB88_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v4
-; SI-NEXT: s_movk_i32 s6, 0x300
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_or_b32_e32 v2, v7, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_or_b32_e32 v7, v2, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_or_b32_e32 v2, v58, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_movk_i32 s6, 0x300
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT: v_or_b32_e32 v0, v57, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v3, v62, v3
+; SI-NEXT: s_mov_b32 s7, 0x3000000
+; SI-NEXT: v_add_i32_e32 v37, vcc, s7, v0
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v37
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v16
; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT: v_or_b32_e32 v5, v47, v5
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v0, v5, v0
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v9
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v11
-; SI-NEXT: v_or_b32_e32 v5, v43, v5
-; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT: v_or_b32_e32 v9, v45, v9
+; SI-NEXT: s_waitcnt vmcnt(13)
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_or_b32_e32 v6, v26, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v5, v1, v5
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v5, vcc, 0x300, v5
; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_or_b32_e32 v5, v9, v5
-; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v20
-; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT: v_or_b32_e32 v6, v6, v9
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v6
-; SI-NEXT: v_or_b32_e32 v8, v54, v8
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v58
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v57
-; SI-NEXT: v_or_b32_e32 v12, v63, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_add_i32_e32 v12, vcc, 0x300, v12
-; SI-NEXT: v_or_b32_e32 v13, v62, v13
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_or_b32_e32 v12, v13, v12
-; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v56
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v14, 8, v46
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v44
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_add_i32_e32 v13, vcc, s6, v13
-; SI-NEXT: v_or_b32_e32 v14, v61, v14
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v41
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v42
-; SI-NEXT: v_or_b32_e32 v14, v59, v14
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v14
+; SI-NEXT: v_or_b32_e32 v3, v3, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v39, vcc, s7, v3
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_or_b32_e32 v15, v60, v15
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: s_mov_b32 s7, 0x3000000
-; SI-NEXT: v_add_i32_e32 v4, vcc, s7, v5
-; SI-NEXT: v_add_i32_e32 v5, vcc, s7, v6
-; SI-NEXT: v_add_i32_e32 v6, vcc, s7, v7
-; SI-NEXT: v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v4
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v0
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v10, v2
-; SI-NEXT: v_or_b32_e32 v8, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v2, v56, v2
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v7, vcc, s7, v8
+; SI-NEXT: v_add_i32_e32 v38, vcc, s7, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_or_b32_e32 v1, v18, v1
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_or_b32_e32 v2, v17, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_or_b32_e32 v9, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v8, vcc, s7, v9
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v4, v2
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v10, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v9, vcc, s7, v10
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v4, v46, v4
+; SI-NEXT: v_or_b32_e32 v2, v4, v2
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v35, vcc, s7, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v18, vcc, 3, v18
+; SI-NEXT: v_and_b32_e32 v18, 0xff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v35
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v4, v5, v4
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v11, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v10, vcc, s7, v11
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_or_b32_e32 v5, v7, v5
+; SI-NEXT: v_or_b32_e32 v4, v5, v4
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v36, vcc, s7, v4
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT: v_or_b32_e32 v5, v63, v5
+; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_or_b32_e32 v5, v6, v5
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v33, vcc, s7, v5
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v33
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v15, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v11, vcc, s7, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 8, v15
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT: v_or_b32_e32 v6, v7, v6
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v16, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v16, vcc, s7, v16
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_or_b32_e32 v7, v8, v7
+; SI-NEXT: v_or_b32_e32 v6, v7, v6
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v34, vcc, s7, v6
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v17, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v17, vcc, s7, v17
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v7, v8, v7
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v7, vcc, s6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v18, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v18, vcc, s7, v18
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v18
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v8, v9, v8
+; SI-NEXT: v_or_b32_e32 v7, v8, v7
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v28, vcc, s7, v7
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v28
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v19, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v20, vcc, s7, v19
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v8, v9, v8
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v8, vcc, s6, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v21, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v22, vcc, s7, v21
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v22
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_or_b32_e32 v9, v10, v9
+; SI-NEXT: v_or_b32_e32 v8, v9, v8
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v27, vcc, s7, v8
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v23, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v24, vcc, s7, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v9, v10, v9
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v9, vcc, s6, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v25, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v21, vcc, s7, v25
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_or_b32_e32 v10, v11, v10
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v9, v10, v9
+; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v32
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT: v_add_i32_e32 v30, vcc, s7, v9
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v26, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v25, vcc, s7, v26
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_or_b32_e32 v10, v11, v10
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v10, vcc, s6, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_or_b32_e32 v11, v12, v11
+; SI-NEXT: v_or_b32_e32 v10, v11, v10
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v27, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_or_b32_e32 v11, v12, v11
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v11, vcc, s6, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_or_b32_e32 v12, v13, v12
+; SI-NEXT: v_or_b32_e32 v11, v12, v11
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v23, vcc, s7, v11
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v23
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v28, v2, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v3, vcc, s7, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v26, vcc, s7, v28
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v3
+; SI-NEXT: v_or_b32_e32 v12, v13, v12
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v12, vcc, s6, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_or_b32_e32 v13, v14, v13
+; SI-NEXT: v_or_b32_e32 v12, v13, v12
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v29, vcc, s7, v12
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_or_b32_e32 v29, v2, v1
+; SI-NEXT: v_or_b32_e32 v13, v14, v13
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v13, vcc, s6, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: v_or_b32_e32 v30, v2, v1
-; SI-NEXT: v_add_i32_e32 v2, vcc, s7, v13
-; SI-NEXT: v_add_i32_e32 v13, vcc, s7, v27
-; SI-NEXT: v_add_i32_e32 v27, vcc, s7, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v1, vcc, s7, v12
-; SI-NEXT: v_add_i32_e32 v12, vcc, s6, v29
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_or_b32_e32 v12, v30, v12
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v12, vcc, s7, v12
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v2
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v1
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_or_b32_e32 v13, v14, v13
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v14
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
+; SI-NEXT: v_or_b32_e32 v15, v16, v15
; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v14, vcc, s7, v14
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v22, vcc, s7, v14
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 8, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v15, v16, v15
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT: v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v16, v17, v16
+; SI-NEXT: v_or_b32_e32 v15, v16, v15
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT: v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v16, v17, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_or_b32_e32 v17, v19, v17
+; SI-NEXT: v_or_b32_e32 v16, v17, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v5, vcc, s7, v16
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v17, v19, v17
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v18, v19, v18
+; SI-NEXT: v_or_b32_e32 v17, v18, v17
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v24, vcc, s7, v17
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v18, vcc, 3, v18
+; SI-NEXT: v_and_b32_e32 v18, 0xff, v18
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v18, v19, v18
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v18, vcc, s6, v18
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v19, v23, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: v_add_i32_e32 v28, vcc, s7, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v19, v20, v19
+; SI-NEXT: v_or_b32_e32 v18, v19, v18
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v4, vcc, s7, v18
+; SI-NEXT: v_mov_b32_e32 v18, v5
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v19, v20, v19
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v20, vcc, 3, v20
+; SI-NEXT: v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_or_b32_e32 v20, v21, v20
+; SI-NEXT: v_or_b32_e32 v25, v20, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v19, v23, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: v_add_i32_e32 v23, vcc, s7, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 8, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v19, v20, v19
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v20, vcc, 3, v20
+; SI-NEXT: v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_or_b32_e32 v20, v21, v20
+; SI-NEXT: v_or_b32_e32 v26, v20, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v3, vcc, s7, v26
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v19, v29, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: v_add_i32_e32 v29, vcc, s7, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v19, v20, v19
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_add_i32_e32 v20, vcc, 3, v20
+; SI-NEXT: v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_or_b32_e32 v20, v21, v20
+; SI-NEXT: v_or_b32_e32 v31, v20, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v21, vcc, s7, v15
+; SI-NEXT: v_add_i32_e32 v31, vcc, s7, v31
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v32, v20, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v32
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v19, v30, v19
-; SI-NEXT: v_or_b32_e32 v15, v19, v15
-; SI-NEXT: v_add_i32_e32 v19, vcc, s7, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v48, v20, v19
+; SI-NEXT: v_or_b32_e32 v0, v48, v0
+; SI-NEXT: v_add_i32_e32 v16, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v19, vcc, s7, v13
+; SI-NEXT: v_add_i32_e32 v13, vcc, s7, v25
+; SI-NEXT: v_add_i32_e32 v20, vcc, s7, v10
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v9, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v59
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v9
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 8, v30
-; SI-NEXT: v_or_b32_e32 v15, v30, v15
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v15, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v30
-; SI-NEXT: v_and_b32_e32 v30, 0xff, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_or_b32_e32 v30, v31, v30
-; SI-NEXT: v_or_b32_e32 v15, v30, v15
-; SI-NEXT: v_add_i32_e32 v30, vcc, s7, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v51, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v47
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v51
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v15, v31, v15
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v6, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v31, vcc, 3, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xff, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_or_b32_e32 v31, v32, v31
-; SI-NEXT: v_or_b32_e32 v15, v31, v15
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s7, v15
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v12, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v43
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v31, vcc, 3, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xff, v31
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v32
-; SI-NEXT: v_or_b32_e32 v31, v32, v31
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v31, vcc, s6, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v48, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v61
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v32, vcc, 3, v32
-; SI-NEXT: v_and_b32_e32 v32, 0xff, v32
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; SI-NEXT: v_or_b32_e32 v32, v33, v32
-; SI-NEXT: v_or_b32_e32 v31, v32, v31
-; SI-NEXT: v_add_i32_e32 v31, vcc, s7, v31
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v19
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v23
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v11, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_or_b32_e32 v0, v40, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v7, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v60
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v8, vcc, s7, v0
+; SI-NEXT: v_alignbit_b32 v0, v11, v48, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v12, v6, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v14
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v51, v15, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v9, v16, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v31, v3, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v13, v4, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v12
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v24, v5, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v21, v22, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v13
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v19, v29, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v23, v20, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v21
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v30, v27, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v28, v34, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v33, v36, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v35, v38, 16
+; SI-NEXT: v_alignbit_b32 v1, v8, v7, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v7
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v5
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v0
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v37, v39, 16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
; SI-NEXT: .LBB88_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v12
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v52
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v41
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v45
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v48
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v4, v4, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v5, v6, v5
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v51
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v16
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_or_b32_e32 v8, v8, v10
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v17
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v49
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v52
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v31
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v50
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v14, v14, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v16, v16, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v25
+; SI-NEXT: v_or_b32_e32 v17, v17, v18
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v29
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v54
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v55
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_or_b32_e32 v18, v18, v21
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v53
+; SI-NEXT: v_or_b32_e32 v19, v19, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v51
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v40
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v34
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v42
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v44
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
@@ -142323,27 +138564,21 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v53
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v34
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v33
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v55
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v38
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v36
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v33
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v38
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v35
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v39
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v37
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -146671,1516 +142906,1516 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; SI-NEXT: v_writelane_b32 v56, s27, 2
; SI-NEXT: v_writelane_b32 v56, s26, 3
; SI-NEXT: v_writelane_b32 v56, s25, 4
-; SI-NEXT: v_writelane_b32 v56, s24, 5
-; SI-NEXT: v_writelane_b32 v56, s23, 6
-; SI-NEXT: v_writelane_b32 v56, s22, 7
-; SI-NEXT: v_writelane_b32 v56, s21, 8
-; SI-NEXT: v_writelane_b32 v56, s4, 9
+; SI-NEXT: v_writelane_b32 v56, s23, 5
+; SI-NEXT: v_writelane_b32 v56, s22, 6
+; SI-NEXT: v_writelane_b32 v56, s21, 7
+; SI-NEXT: v_writelane_b32 v56, s19, 8
+; SI-NEXT: v_writelane_b32 v56, s18, 9
+; SI-NEXT: v_writelane_b32 v56, s17, 10
+; SI-NEXT: v_writelane_b32 v56, s16, 11
+; SI-NEXT: v_writelane_b32 v56, s4, 12
; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: v_writelane_b32 v56, s4, 10
+; SI-NEXT: v_writelane_b32 v56, s4, 13
; SI-NEXT: v_readfirstlane_b32 s4, v28
-; SI-NEXT: v_writelane_b32 v56, s4, 11
+; SI-NEXT: v_writelane_b32 v56, s4, 14
; SI-NEXT: v_readfirstlane_b32 s4, v27
-; SI-NEXT: v_writelane_b32 v56, s4, 12
+; SI-NEXT: v_writelane_b32 v56, s4, 15
; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: v_writelane_b32 v56, s4, 13
+; SI-NEXT: v_writelane_b32 v56, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v25
-; SI-NEXT: v_writelane_b32 v56, s4, 14
+; SI-NEXT: v_writelane_b32 v56, s4, 17
; SI-NEXT: v_readfirstlane_b32 s4, v24
-; SI-NEXT: v_writelane_b32 v56, s4, 15
+; SI-NEXT: v_writelane_b32 v56, s4, 18
; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:332
-; SI-NEXT: s_mov_b32 s6, s19
-; SI-NEXT: v_writelane_b32 v56, s4, 16
+; SI-NEXT: v_writelane_b32 v56, s4, 19
; SI-NEXT: v_readfirstlane_b32 s4, v22
-; SI-NEXT: v_readfirstlane_b32 s19, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:328
-; SI-NEXT: v_writelane_b32 v56, s4, 17
+; SI-NEXT: v_writelane_b32 v56, s4, 20
; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:324
-; SI-NEXT: v_writelane_b32 v56, s4, 18
+; SI-NEXT: v_writelane_b32 v56, s4, 21
; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: v_writelane_b32 v56, s4, 19
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:332
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:328
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:324
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:320
+; SI-NEXT: v_writelane_b32 v56, s4, 22
; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:320
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:316
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:308
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:304
-; SI-NEXT: v_writelane_b32 v56, s4, 20
-; SI-NEXT: v_readfirstlane_b32 s75, v15
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: v_readfirstlane_b32 s27, v12
-; SI-NEXT: v_readfirstlane_b32 s52, v0
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:244
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
-; SI-NEXT: v_readfirstlane_b32 s89, v10
-; SI-NEXT: v_readfirstlane_b32 s62, v9
-; SI-NEXT: v_readfirstlane_b32 s92, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:300
-; SI-NEXT: v_readfirstlane_b32 s88, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
-; SI-NEXT: v_readfirstlane_b32 s59, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292
-; SI-NEXT: v_readfirstlane_b32 s77, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288
-; SI-NEXT: s_mov_b32 s7, s20
-; SI-NEXT: v_readfirstlane_b32 s79, v8
-; SI-NEXT: v_readfirstlane_b32 s12, v7
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:284
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:264
-; SI-NEXT: v_readfirstlane_b32 s46, v6
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:316
+; SI-NEXT: v_writelane_b32 v56, s4, 23
+; SI-NEXT: v_readfirstlane_b32 s4, v18
+; SI-NEXT: v_writelane_b32 v56, s4, 24
+; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: v_writelane_b32 v56, s4, 25
+; SI-NEXT: v_readfirstlane_b32 s4, v16
+; SI-NEXT: v_writelane_b32 v56, s4, 26
+; SI-NEXT: v_readfirstlane_b32 s4, v15
+; SI-NEXT: v_writelane_b32 v56, s4, 27
+; SI-NEXT: v_readfirstlane_b32 s4, v14
+; SI-NEXT: v_writelane_b32 v56, s4, 28
+; SI-NEXT: v_readfirstlane_b32 s4, v11
+; SI-NEXT: v_writelane_b32 v56, s4, 29
+; SI-NEXT: v_readfirstlane_b32 s4, v9
+; SI-NEXT: v_writelane_b32 v56, s4, 30
+; SI-NEXT: v_readfirstlane_b32 s4, v8
+; SI-NEXT: v_writelane_b32 v56, s4, 31
+; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:284
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264
+; SI-NEXT: v_writelane_b32 v56, s4, 32
+; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:280
-; SI-NEXT: v_readfirstlane_b32 s22, v5
+; SI-NEXT: v_readfirstlane_b32 s98, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:276
-; SI-NEXT: v_readfirstlane_b32 s36, v4
+; SI-NEXT: v_readfirstlane_b32 s99, v4
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
-; SI-NEXT: v_readfirstlane_b32 s20, v3
+; SI-NEXT: v_writelane_b32 v56, s4, 33
+; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:268
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:308
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292
+; SI-NEXT: v_readfirstlane_b32 s96, v13
+; SI-NEXT: v_readfirstlane_b32 s97, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:304
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:300
+; SI-NEXT: v_readfirstlane_b32 s76, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:296
+; SI-NEXT: v_writelane_b32 v56, s4, 34
+; SI-NEXT: v_readfirstlane_b32 s77, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260
; SI-NEXT: ; implicit-def: $vgpr47 : SGPR spill to VGPR lane
-; SI-NEXT: ; implicit-def: $vgpr46 : SGPR spill to VGPR lane
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v23
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v22
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:256
+; SI-NEXT: v_readfirstlane_b32 s81, v21
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:224
+; SI-NEXT: v_readfirstlane_b32 s4, v1
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248
-; SI-NEXT: v_readfirstlane_b32 vcc_hi, v2
-; SI-NEXT: v_readfirstlane_b32 s83, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:236
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:232
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260
-; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:224
-; SI-NEXT: v_readfirstlane_b32 s51, v19
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:220
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:216
-; SI-NEXT: v_readfirstlane_b32 s10, v25
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:208
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248
+; SI-NEXT: v_writelane_b32 v56, s4, 35
+; SI-NEXT: v_readfirstlane_b32 s4, v0
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:244
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
+; SI-NEXT: v_readfirstlane_b32 s82, v23
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:216
+; SI-NEXT: v_readfirstlane_b32 s80, v20
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:232
+; SI-NEXT: v_readfirstlane_b32 s83, v19
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:212
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:208
+; SI-NEXT: v_writelane_b32 v56, s4, 36
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s8, v22
-; SI-NEXT: v_writelane_b32 v56, s8, 21
-; SI-NEXT: v_readfirstlane_b32 s8, v20
-; SI-NEXT: v_writelane_b32 v56, s8, 22
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:188
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:184
-; SI-NEXT: v_readfirstlane_b32 s39, v12
-; SI-NEXT: v_readfirstlane_b32 s31, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:64
-; SI-NEXT: v_readfirstlane_b32 s5, v24
-; SI-NEXT: v_readfirstlane_b32 s8, v17
-; SI-NEXT: v_writelane_b32 v56, s8, 23
-; SI-NEXT: v_readfirstlane_b32 s8, v14
-; SI-NEXT: v_writelane_b32 v56, s8, 24
-; SI-NEXT: v_readfirstlane_b32 s8, v13
-; SI-NEXT: v_writelane_b32 v56, s8, 25
-; SI-NEXT: v_readfirstlane_b32 s8, v11
-; SI-NEXT: v_writelane_b32 v56, s8, 26
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156
+; SI-NEXT: v_readfirstlane_b32 s21, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:152
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:172
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
-; SI-NEXT: v_readfirstlane_b32 s8, v8
-; SI-NEXT: v_readfirstlane_b32 s37, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108
+; SI-NEXT: v_readfirstlane_b32 s44, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
-; SI-NEXT: v_writelane_b32 v56, s8, 27
-; SI-NEXT: v_readfirstlane_b32 s8, v6
-; SI-NEXT: v_writelane_b32 v56, s8, 28
-; SI-NEXT: v_readfirstlane_b32 s8, v5
-; SI-NEXT: v_writelane_b32 v56, s8, 29
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_writelane_b32 v56, s8, 30
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
+; SI-NEXT: v_readfirstlane_b32 s46, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
+; SI-NEXT: v_readfirstlane_b32 s72, v6
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT: v_readfirstlane_b32 s73, v5
+; SI-NEXT: v_readfirstlane_b32 s40, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:128
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:164
+; SI-NEXT: v_readfirstlane_b32 s69, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
+; SI-NEXT: v_readfirstlane_b32 s47, v15
+; SI-NEXT: v_readfirstlane_b32 s55, v9
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:164
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:180
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:196
+; SI-NEXT: v_readfirstlane_b32 s57, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196
; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:192
-; SI-NEXT: v_writelane_b32 v56, s8, 31
-; SI-NEXT: v_readfirstlane_b32 s85, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
-; SI-NEXT: v_readfirstlane_b32 s35, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
-; SI-NEXT: v_readfirstlane_b32 s48, v18
-; SI-NEXT: v_readfirstlane_b32 s63, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:44
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:40
-; SI-NEXT: v_readfirstlane_b32 s38, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
-; SI-NEXT: v_readfirstlane_b32 s30, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT: v_readfirstlane_b32 s16, v12
+; SI-NEXT: v_readfirstlane_b32 s54, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s45, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:96
+; SI-NEXT: v_readfirstlane_b32 s18, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:180
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
+; SI-NEXT: v_readfirstlane_b32 s60, v24
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:28
; SI-NEXT: s_waitcnt expcnt(3)
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52
+; SI-NEXT: v_readfirstlane_b32 s84, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
+; SI-NEXT: v_readfirstlane_b32 s90, v18
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:68
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:64
+; SI-NEXT: v_readfirstlane_b32 s23, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:20
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16
+; SI-NEXT: v_readfirstlane_b32 s65, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:12
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8
+; SI-NEXT: v_readfirstlane_b32 s70, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
+; SI-NEXT: v_readfirstlane_b32 s37, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:52
; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
-; SI-NEXT: v_readfirstlane_b32 s82, v1
+; SI-NEXT: v_readfirstlane_b32 s56, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:44
+; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:40
+; SI-NEXT: v_readfirstlane_b32 s17, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
+; SI-NEXT: v_readfirstlane_b32 s62, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:80
-; SI-NEXT: v_readfirstlane_b32 s78, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:96
-; SI-NEXT: v_readfirstlane_b32 s86, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:12
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s61, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
+; SI-NEXT: v_readfirstlane_b32 s94, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32
-; SI-NEXT: v_readfirstlane_b32 s84, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:20
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16
-; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: v_readfirstlane_b32 s8, v53
-; SI-NEXT: v_readfirstlane_b32 s24, v12
-; SI-NEXT: v_readfirstlane_b32 s29, v52
-; SI-NEXT: v_readfirstlane_b32 s97, v29
-; SI-NEXT: v_readfirstlane_b32 s98, v20
-; SI-NEXT: v_readfirstlane_b32 s49, v25
-; SI-NEXT: v_readfirstlane_b32 s73, v11
-; SI-NEXT: v_readfirstlane_b32 s96, v33
-; SI-NEXT: v_readfirstlane_b32 s26, v31
-; SI-NEXT: v_readfirstlane_b32 s94, v14
-; SI-NEXT: v_readfirstlane_b32 s72, v7
-; SI-NEXT: v_readfirstlane_b32 s58, v39
-; SI-NEXT: v_readfirstlane_b32 s81, v4
-; SI-NEXT: v_readfirstlane_b32 s60, v37
-; SI-NEXT: v_readfirstlane_b32 s47, v3
-; SI-NEXT: v_readfirstlane_b32 s76, v38
-; SI-NEXT: v_readfirstlane_b32 s91, v5
-; SI-NEXT: v_readfirstlane_b32 s21, v35
-; SI-NEXT: v_readfirstlane_b32 s57, v36
-; SI-NEXT: v_readfirstlane_b32 s87, v8
-; SI-NEXT: v_readfirstlane_b32 s99, v34
-; SI-NEXT: v_readfirstlane_b32 s90, v6
-; SI-NEXT: v_readfirstlane_b32 s28, v13
-; SI-NEXT: v_readfirstlane_b32 s25, v32
-; SI-NEXT: v_readfirstlane_b32 s74, v17
-; SI-NEXT: v_readfirstlane_b32 s93, v30
-; SI-NEXT: v_readfirstlane_b32 s23, v22
-; SI-NEXT: v_readfirstlane_b32 s50, v26
-; SI-NEXT: v_readfirstlane_b32 s95, v27
-; SI-NEXT: v_readfirstlane_b32 s34, v24
-; SI-NEXT: v_readfirstlane_b32 s45, v9
-; SI-NEXT: v_readfirstlane_b32 s15, v49
-; SI-NEXT: v_readfirstlane_b32 s56, v10
-; SI-NEXT: v_readfirstlane_b32 s13, v51
+; SI-NEXT: v_readfirstlane_b32 s88, v8
+; SI-NEXT: v_readfirstlane_b32 s13, v11
+; SI-NEXT: v_readfirstlane_b32 s66, v33
+; SI-NEXT: v_readfirstlane_b32 s78, v7
+; SI-NEXT: v_readfirstlane_b32 s14, v34
+; SI-NEXT: v_readfirstlane_b32 s15, v6
+; SI-NEXT: v_readfirstlane_b32 s93, v39
+; SI-NEXT: v_readfirstlane_b32 s39, v35
+; SI-NEXT: v_readfirstlane_b32 s49, v4
+; SI-NEXT: v_readfirstlane_b32 s51, v37
+; SI-NEXT: v_writelane_b32 v47, s51, 0
+; SI-NEXT: v_writelane_b32 v47, s54, 1
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s9, v15
-; SI-NEXT: v_readfirstlane_b32 s69, v55
-; SI-NEXT: v_readfirstlane_b32 s68, v21
-; SI-NEXT: v_writelane_b32 v47, s68, 0
-; SI-NEXT: v_readfirstlane_b32 s70, v40
-; SI-NEXT: v_writelane_b32 v47, s69, 1
+; SI-NEXT: v_readfirstlane_b32 s12, v38
+; SI-NEXT: v_writelane_b32 v47, s83, 2
+; SI-NEXT: v_readfirstlane_b32 s52, v9
+; SI-NEXT: v_writelane_b32 v47, s12, 3
+; SI-NEXT: v_writelane_b32 v47, s52, 4
+; SI-NEXT: v_readfirstlane_b32 s67, v32
+; SI-NEXT: v_readfirstlane_b32 s58, v14
+; SI-NEXT: v_readfirstlane_b32 s59, v26
+; SI-NEXT: v_readfirstlane_b32 s61, v27
+; SI-NEXT: v_readfirstlane_b32 s53, v31
+; SI-NEXT: v_writelane_b32 v47, s53, 5
+; SI-NEXT: v_writelane_b32 v47, s55, 6
+; SI-NEXT: v_writelane_b32 v47, s13, 7
+; SI-NEXT: v_writelane_b32 v47, s67, 8
+; SI-NEXT: v_writelane_b32 v47, s88, 9
+; SI-NEXT: v_writelane_b32 v47, s44, 10
+; SI-NEXT: v_writelane_b32 v47, s45, 11
+; SI-NEXT: v_writelane_b32 v47, s46, 12
+; SI-NEXT: v_writelane_b32 v47, s78, 13
+; SI-NEXT: v_writelane_b32 v47, s66, 14
+; SI-NEXT: v_writelane_b32 v47, s47, 15
+; SI-NEXT: v_writelane_b32 v47, s56, 16
+; SI-NEXT: v_writelane_b32 v47, s57, 17
+; SI-NEXT: v_writelane_b32 v47, s14, 18
+; SI-NEXT: v_writelane_b32 v47, s58, 19
+; SI-NEXT: v_writelane_b32 v47, s59, 20
+; SI-NEXT: v_writelane_b32 v47, s17, 21
+; SI-NEXT: v_writelane_b32 v47, s60, 22
+; SI-NEXT: v_writelane_b32 v47, s15, 23
+; SI-NEXT: v_writelane_b32 v47, s61, 24
+; SI-NEXT: v_readfirstlane_b32 s63, v12
+; SI-NEXT: v_writelane_b32 v47, s62, 25
+; SI-NEXT: v_writelane_b32 v47, s63, 26
+; SI-NEXT: v_writelane_b32 v47, s16, 27
+; SI-NEXT: v_readfirstlane_b32 s92, v2
+; SI-NEXT: v_writelane_b32 v47, vcc_lo, 28
+; SI-NEXT: v_writelane_b32 v47, s72, 29
+; SI-NEXT: v_writelane_b32 v47, s73, 30
+; SI-NEXT: v_writelane_b32 v47, s23, 31
+; SI-NEXT: v_readfirstlane_b32 s74, v13
+; SI-NEXT: v_readfirstlane_b32 s95, v48
+; SI-NEXT: v_writelane_b32 v47, s18, 32
+; SI-NEXT: v_readfirstlane_b32 s64, v28
+; SI-NEXT: v_writelane_b32 v47, s74, 33
+; SI-NEXT: v_writelane_b32 v47, s64, 34
+; SI-NEXT: v_readfirstlane_b32 s41, v30
+; SI-NEXT: v_readfirstlane_b32 s19, v41
+; SI-NEXT: v_writelane_b32 v56, s19, 37
+; SI-NEXT: v_readfirstlane_b32 s43, v51
+; SI-NEXT: v_readfirstlane_b32 s42, v18
+; SI-NEXT: v_writelane_b32 v56, s42, 38
+; SI-NEXT: v_writelane_b32 v56, s43, 39
+; SI-NEXT: v_readfirstlane_b32 s6, v42
+; SI-NEXT: v_readfirstlane_b32 s7, v23
+; SI-NEXT: v_writelane_b32 v56, s6, 40
+; SI-NEXT: v_readfirstlane_b32 s75, v52
+; SI-NEXT: v_writelane_b32 v56, s7, 41
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_readfirstlane_b32 s71, v16
-; SI-NEXT: v_writelane_b32 v47, s70, 2
+; SI-NEXT: v_readfirstlane_b32 s79, v0
+; SI-NEXT: v_writelane_b32 v56, s75, 42
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_readfirstlane_b32 s80, v18
-; SI-NEXT: v_writelane_b32 v47, s71, 3
-; SI-NEXT: v_writelane_b32 v47, s80, 4
-; SI-NEXT: v_writelane_b32 v47, s8, 5
-; SI-NEXT: v_writelane_b32 v47, s9, 6
-; SI-NEXT: v_writelane_b32 v47, s24, 7
-; SI-NEXT: v_writelane_b32 v47, s13, 8
-; SI-NEXT: v_writelane_b32 v47, s29, 9
+; SI-NEXT: v_readfirstlane_b32 s89, v22
+; SI-NEXT: v_writelane_b32 v56, s79, 43
+; SI-NEXT: v_readfirstlane_b32 s91, v53
+; SI-NEXT: v_writelane_b32 v56, s89, 44
+; SI-NEXT: v_writelane_b32 v56, s91, 45
+; SI-NEXT: v_writelane_b32 v56, s92, 46
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_readfirstlane_b32 s8, v54
+; SI-NEXT: v_writelane_b32 v56, s93, 47
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s14, v1
-; SI-NEXT: v_writelane_b32 v47, s11, 10
-; SI-NEXT: v_writelane_b32 v47, s14, 11
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s40, v50
-; SI-NEXT: v_writelane_b32 v47, s15, 12
-; SI-NEXT: v_writelane_b32 v47, s40, 13
+; SI-NEXT: v_readfirstlane_b32 s9, v20
+; SI-NEXT: v_writelane_b32 v56, s8, 48
+; SI-NEXT: v_writelane_b32 v56, s9, 49
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_readfirstlane_b32 s42, v2
-; SI-NEXT: v_writelane_b32 v47, s56, 14
-; SI-NEXT: v_writelane_b32 v47, s42, 15
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s43, v48
-; SI-NEXT: v_writelane_b32 v47, s58, 16
-; SI-NEXT: v_writelane_b32 v47, s43, 17
-; SI-NEXT: v_writelane_b32 v47, s45, 18
-; SI-NEXT: v_writelane_b32 v47, s47, 19
-; SI-NEXT: v_writelane_b32 v47, s60, 20
-; SI-NEXT: v_writelane_b32 v47, s76, 21
-; SI-NEXT: v_writelane_b32 v47, s72, 22
-; SI-NEXT: v_writelane_b32 v47, s91, 23
-; SI-NEXT: v_writelane_b32 v47, s21, 24
-; SI-NEXT: v_writelane_b32 v47, s57, 25
-; SI-NEXT: v_readfirstlane_b32 s41, v54
-; SI-NEXT: v_writelane_b32 v47, s81, 26
-; SI-NEXT: v_writelane_b32 v56, s41, 32
-; SI-NEXT: v_writelane_b32 v47, s87, 27
-; SI-NEXT: v_writelane_b32 v56, s18, 33
-; SI-NEXT: v_writelane_b32 v47, s96, 28
-; SI-NEXT: v_writelane_b32 v56, s6, 34
-; SI-NEXT: v_writelane_b32 v47, s99, 29
-; SI-NEXT: v_writelane_b32 v56, s17, 35
-; SI-NEXT: v_writelane_b32 v47, s90, 30
-; SI-NEXT: v_writelane_b32 v56, s16, 36
-; SI-NEXT: v_writelane_b32 v47, s28, 31
-; SI-NEXT: v_writelane_b32 v56, s7, 37
-; SI-NEXT: v_writelane_b32 v47, s26, 32
-; SI-NEXT: v_writelane_b32 v56, s92, 38
-; SI-NEXT: v_writelane_b32 v47, s37, 33
-; SI-NEXT: v_writelane_b32 v56, s44, 39
-; SI-NEXT: v_writelane_b32 v47, s25, 34
-; SI-NEXT: v_writelane_b32 v56, s59, 40
-; SI-NEXT: v_writelane_b32 v47, s78, 35
-; SI-NEXT: v_writelane_b32 v56, s27, 41
-; SI-NEXT: v_writelane_b32 v47, s73, 36
-; SI-NEXT: v_writelane_b32 v56, s62, 42
-; SI-NEXT: v_writelane_b32 v47, s74, 37
-; SI-NEXT: v_writelane_b32 v56, s79, 43
-; SI-NEXT: v_writelane_b32 v47, s97, 38
-; SI-NEXT: v_writelane_b32 v56, s22, 44
-; SI-NEXT: v_writelane_b32 v47, s35, 39
-; SI-NEXT: v_writelane_b32 v56, s75, 45
-; SI-NEXT: v_writelane_b32 v47, s39, 40
-; SI-NEXT: v_writelane_b32 v56, s36, 46
-; SI-NEXT: v_writelane_b32 v47, s85, 41
-; SI-NEXT: v_writelane_b32 v56, s19, 47
-; SI-NEXT: v_writelane_b32 v47, s93, 42
-; SI-NEXT: v_writelane_b32 v56, s77, 48
-; SI-NEXT: v_writelane_b32 v47, s63, 43
-; SI-NEXT: v_writelane_b32 v56, s88, 49
-; SI-NEXT: v_writelane_b32 v47, s86, 44
-; SI-NEXT: v_writelane_b32 v56, s12, 50
-; SI-NEXT: v_writelane_b32 v47, s94, 45
-; SI-NEXT: v_writelane_b32 v56, s89, 51
-; SI-NEXT: v_writelane_b32 v47, s38, 46
-; SI-NEXT: v_writelane_b32 v56, s46, 52
-; SI-NEXT: v_writelane_b32 v47, s61, 47
-; SI-NEXT: v_writelane_b32 v56, s20, 53
-; SI-NEXT: v_writelane_b32 v47, s23, 48
-; SI-NEXT: v_writelane_b32 v56, vcc_lo, 54
-; SI-NEXT: v_writelane_b32 v47, s50, 49
-; SI-NEXT: v_writelane_b32 v56, vcc_hi, 55
-; SI-NEXT: v_writelane_b32 v47, s82, 50
+; SI-NEXT: v_readfirstlane_b32 s31, v16
+; SI-NEXT: v_writelane_b32 v56, s95, 50
+; SI-NEXT: v_writelane_b32 v56, s31, 51
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_readfirstlane_b32 s34, v1
+; SI-NEXT: v_writelane_b32 v56, s80, 52
+; SI-NEXT: v_writelane_b32 v47, s40, 35
+; SI-NEXT: v_readfirstlane_b32 s36, v49
+; SI-NEXT: v_writelane_b32 v56, s34, 53
+; SI-NEXT: v_writelane_b32 v47, s41, 36
+; SI-NEXT: v_readfirstlane_b32 s68, v10
+; SI-NEXT: v_writelane_b32 v56, s36, 54
+; SI-NEXT: v_writelane_b32 v47, s84, 37
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_readfirstlane_b32 s10, v50
+; SI-NEXT: v_writelane_b32 v56, s81, 55
+; SI-NEXT: v_writelane_b32 v47, s68, 38
+; SI-NEXT: v_readfirstlane_b32 s38, v5
+; SI-NEXT: v_writelane_b32 v56, s10, 56
+; SI-NEXT: v_writelane_b32 v47, s65, 39
+; SI-NEXT: v_writelane_b32 v56, s38, 57
+; SI-NEXT: v_writelane_b32 v47, s90, 40
+; SI-NEXT: v_readfirstlane_b32 s11, v17
+; SI-NEXT: v_writelane_b32 v56, s39, 58
+; SI-NEXT: v_writelane_b32 v47, s69, 41
+; SI-NEXT: v_readfirstlane_b32 s48, v36
+; SI-NEXT: v_writelane_b32 v56, s11, 59
+; SI-NEXT: v_writelane_b32 v47, s94, 42
+; SI-NEXT: v_writelane_b32 v56, s48, 60
+; SI-NEXT: v_writelane_b32 v47, s70, 43
+; SI-NEXT: v_readfirstlane_b32 s35, v19
+; SI-NEXT: v_writelane_b32 v56, s49, 61
+; SI-NEXT: v_writelane_b32 v47, s37, 44
+; SI-NEXT: v_readfirstlane_b32 s71, v15
+; SI-NEXT: v_readfirstlane_b32 s50, v3
+; SI-NEXT: v_readfirstlane_b32 s22, v55
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s53, v28
-; SI-NEXT: v_writelane_b32 v56, s52, 56
-; SI-NEXT: v_writelane_b32 v47, s31, 51
-; SI-NEXT: v_readfirstlane_b32 s54, v43
-; SI-NEXT: v_writelane_b32 v56, s53, 57
-; SI-NEXT: v_writelane_b32 v47, s95, 52
+; SI-NEXT: v_readfirstlane_b32 s85, v24
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s55, v44
-; SI-NEXT: v_writelane_b32 v56, s54, 58
-; SI-NEXT: v_writelane_b32 v47, s30, 53
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s64, v23
-; SI-NEXT: v_writelane_b32 v56, s55, 59
-; SI-NEXT: v_writelane_b32 v47, s98, 54
-; SI-NEXT: v_readfirstlane_b32 s65, v41
-; SI-NEXT: v_writelane_b32 v56, s64, 60
-; SI-NEXT: v_writelane_b32 v47, s48, 55
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s66, v42
-; SI-NEXT: v_writelane_b32 v56, s65, 61
-; SI-NEXT: v_writelane_b32 v47, s84, 56
-; SI-NEXT: v_readfirstlane_b32 s67, v19
-; SI-NEXT: v_writelane_b32 v56, s66, 62
-; SI-NEXT: v_writelane_b32 v47, s49, 57
-; SI-NEXT: v_writelane_b32 v56, s67, 63
-; SI-NEXT: v_writelane_b32 v47, s34, 58
+; SI-NEXT: v_readfirstlane_b32 s25, v40
+; SI-NEXT: v_readfirstlane_b32 s28, v21
+; SI-NEXT: v_readfirstlane_b32 s87, v25
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v43
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s26, v29
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s27, v44
+; SI-NEXT: v_writelane_b32 v56, s82, 62
+; SI-NEXT: v_writelane_b32 v47, s35, 45
+; SI-NEXT: v_writelane_b32 v56, s50, 63
+; SI-NEXT: v_writelane_b32 v47, s71, 46
; SI-NEXT: s_cbranch_scc0 .LBB89_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_writelane_b32 v46, s83, 7
-; SI-NEXT: v_writelane_b32 v46, s4, 8
-; SI-NEXT: s_and_b32 s4, s16, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_mov_b32 s83, s51
-; SI-NEXT: s_mov_b32 s51, s10
-; SI-NEXT: s_mov_b32 s10, s5
-; SI-NEXT: s_lshl_b32 s5, s17, 24
+; SI-NEXT: v_readlane_b32 s4, v56, 11
+; SI-NEXT: v_readlane_b32 s5, v56, 10
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: v_writelane_b32 v47, s21, 47
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s5, v56, 4
+; SI-NEXT: v_writelane_b32 v47, s4, 48
+; SI-NEXT: s_and_b32 s4, s24, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s5, v56, 34
+; SI-NEXT: v_writelane_b32 v47, s4, 49
+; SI-NEXT: s_and_b32 s4, s77, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s5, v56, 29
+; SI-NEXT: v_writelane_b32 v47, s4, 50
+; SI-NEXT: s_and_b32 s4, s76, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 51
+; SI-NEXT: v_readlane_b32 s4, v56, 24
+; SI-NEXT: s_and_b32 s5, s4, 0xff
+; SI-NEXT: v_readlane_b32 s4, v56, 23
+; SI-NEXT: s_lshl_b32 s4, s4, 8
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_writelane_b32 v47, s4, 52
+; SI-NEXT: v_readlane_b32 s4, v56, 16
+; SI-NEXT: v_readlane_b32 s5, v56, 15
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 53
+; SI-NEXT: s_and_b32 s4, s7, 0xff
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 54
+; SI-NEXT: s_and_b32 s4, s9, 0xff
+; SI-NEXT: s_lshl_b32 s5, s8, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 55
+; SI-NEXT: s_and_b32 s4, s11, 0xff
+; SI-NEXT: s_lshl_b32 s5, s10, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 56
+; SI-NEXT: s_and_b32 s4, s13, 0xff
+; SI-NEXT: s_lshl_b32 s5, s12, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 57
+; SI-NEXT: s_and_b32 s4, s15, 0xff
+; SI-NEXT: s_lshl_b32 s5, s14, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_writelane_b32 v47, s4, 58
+; SI-NEXT: s_and_b32 s4, s68, 0xff
+; SI-NEXT: s_lshl_b32 s5, s41, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_writelane_b32 v47, s4, 59
+; SI-NEXT: s_and_b32 s4, s71, 0xff
+; SI-NEXT: s_lshl_b32 s5, s35, 8
+; SI-NEXT: s_or_b32 s71, s4, s5
+; SI-NEXT: s_and_b32 s4, s37, 0xff
+; SI-NEXT: s_lshl_b32 s5, s70, 8
+; SI-NEXT: s_or_b32 s70, s4, s5
+; SI-NEXT: s_and_b32 s4, s69, 0xff
+; SI-NEXT: s_lshl_b32 s5, s40, 8
+; SI-NEXT: s_or_b32 s69, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s6, 24
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_readlane_b32 s5, v56, 8
+; SI-NEXT: s_lshl_b32 s5, s16, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s5, v56, 7
; SI-NEXT: v_writelane_b32 v47, s4, 60
-; SI-NEXT: s_and_b32 s4, s7, 0xff
+; SI-NEXT: s_and_b32 s4, s20, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 9
-; SI-NEXT: v_readlane_b32 s4, v56, 7
-; SI-NEXT: s_and_b32 s5, s4, 0xff
+; SI-NEXT: s_or_b32 s5, s4, s5
; SI-NEXT: v_readlane_b32 s4, v56, 6
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s4, 24
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s4, v56, 5
-; SI-NEXT: v_writelane_b32 v47, s5, 62
-; SI-NEXT: s_and_b32 s5, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 4
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s4, 24
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s4, v56, 3
-; SI-NEXT: v_writelane_b32 v47, s5, 61
-; SI-NEXT: s_and_b32 s5, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 2
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s4, 24
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s4, v56, 1
-; SI-NEXT: v_writelane_b32 v47, s5, 63
-; SI-NEXT: s_and_b32 s5, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 0
-; SI-NEXT: s_lshl_b32 s6, s4, 8
-; SI-NEXT: s_or_b32 s4, s5, s6
-; SI-NEXT: s_and_b32 s6, s52, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, vcc_lo, 24
-; SI-NEXT: v_writelane_b32 v46, s4, 10
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s6, 1
-; SI-NEXT: s_and_b32 s6, vcc_hi, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s20, 24
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s6, 0
-; SI-NEXT: s_and_b32 s6, s36, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s22, 24
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s6, 2
-; SI-NEXT: s_and_b32 s6, s46, 0xff
-; SI-NEXT: s_lshl_b32 s7, s12, 8
-; SI-NEXT: s_or_b32 s4, s6, s7
-; SI-NEXT: s_and_b32 s6, s79, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s62, 24
-; SI-NEXT: v_writelane_b32 v46, s4, 11
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s6, 4
-; SI-NEXT: s_and_b32 s6, s89, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s77, 24
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s6, 3
-; SI-NEXT: s_and_b32 s6, s27, 0xff
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: v_readlane_b32 s6, v56, 5
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_or_b32 s7, s6, s4
+; SI-NEXT: v_readlane_b32 s4, v56, 9
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: v_readlane_b32 s6, v56, 8
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_writelane_b32 v47, s7, 61
+; SI-NEXT: s_or_b32 s4, s6, s4
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: v_readlane_b32 s6, v56, 1
+; SI-NEXT: v_readlane_b32 s7, v56, 0
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_or_b32 s7, s6, s7
+; SI-NEXT: v_readlane_b32 s6, v56, 36
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: v_readlane_b32 s8, v56, 35
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s59, 24
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s6, 5
-; SI-NEXT: s_and_b32 s6, s88, 0xff
-; SI-NEXT: s_lshl_b32 s7, s75, 8
-; SI-NEXT: s_or_b32 s4, s6, s7
-; SI-NEXT: s_and_b32 s6, s44, 0xff
+; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_or_b32 s9, s8, s6
+; SI-NEXT: v_readlane_b32 s6, v56, 3
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: v_readlane_b32 s8, v56, 2
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s92, 24
-; SI-NEXT: v_writelane_b32 v46, s4, 12
-; SI-NEXT: s_or_b32 s4, s7, s6
-; SI-NEXT: v_writelane_b32 v46, s4, 6
-; SI-NEXT: s_and_b32 s7, s19, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 20
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s12, s4, 24
-; SI-NEXT: v_readlane_b32 s4, v56, 19
-; SI-NEXT: s_or_b32 s7, s12, s7
-; SI-NEXT: s_and_b32 s12, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 18
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s18, s4, 24
-; SI-NEXT: v_readlane_b32 s4, v56, 17
-; SI-NEXT: s_or_b32 s18, s18, s12
-; SI-NEXT: s_and_b32 s12, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 16
-; SI-NEXT: s_lshl_b32 s19, s4, 8
-; SI-NEXT: v_readlane_b32 s4, v56, 15
-; SI-NEXT: s_or_b32 s6, s12, s19
-; SI-NEXT: s_and_b32 s12, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 14
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s19, s4, 24
-; SI-NEXT: v_readlane_b32 s4, v56, 13
-; SI-NEXT: s_or_b32 s16, s19, s12
-; SI-NEXT: s_and_b32 s12, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 12
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s20, s4, 24
-; SI-NEXT: v_readlane_b32 s4, v56, 11
-; SI-NEXT: s_or_b32 s5, s20, s12
-; SI-NEXT: s_and_b32 s12, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v56, 10
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s20, s4, 24
-; SI-NEXT: s_or_b32 s4, s20, s12
-; SI-NEXT: v_readlane_b32 s12, v56, 9
+; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: v_writelane_b32 v47, s9, 62
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: v_readlane_b32 s8, v56, 33
+; SI-NEXT: v_readlane_b32 s9, v56, 32
+; SI-NEXT: s_and_b32 s8, s8, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 8
+; SI-NEXT: s_or_b32 s9, s8, s9
+; SI-NEXT: v_readlane_b32 s8, v56, 31
+; SI-NEXT: s_and_b32 s8, s8, 0xff
+; SI-NEXT: v_readlane_b32 s10, v56, 30
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_lshl_b32 s10, s10, 24
+; SI-NEXT: s_or_b32 s11, s10, s8
+; SI-NEXT: s_and_b32 s8, s99, 0xff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_lshl_b32 s10, s98, 24
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: v_writelane_b32 v47, s11, 63
+; SI-NEXT: s_or_b32 s8, s10, s8
+; SI-NEXT: s_or_b32 s9, s9, s11
+; SI-NEXT: v_readlane_b32 s10, v56, 28
+; SI-NEXT: v_readlane_b32 s11, v56, 27
+; SI-NEXT: s_and_b32 s10, s10, 0xff
+; SI-NEXT: s_lshl_b32 s11, s11, 8
+; SI-NEXT: s_or_b32 s11, s10, s11
+; SI-NEXT: v_readlane_b32 s10, v56, 26
+; SI-NEXT: s_and_b32 s10, s10, 0xff
+; SI-NEXT: v_readlane_b32 s12, v56, 25
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_lshl_b32 s12, s12, 24
+; SI-NEXT: s_or_b32 s13, s12, s10
+; SI-NEXT: s_and_b32 s10, s97, 0xff
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_lshl_b32 s12, s96, 24
+; SI-NEXT: ; implicit-def: $vgpr46 : SGPR spill to VGPR lane
+; SI-NEXT: s_or_b32 s10, s12, s10
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: v_readlane_b32 s12, v56, 20
+; SI-NEXT: v_writelane_b32 v46, s13, 0
+; SI-NEXT: s_or_b32 s11, s11, s13
+; SI-NEXT: s_and_b32 s13, s12, 0xff
+; SI-NEXT: v_readlane_b32 s12, v56, 19
+; SI-NEXT: s_lshl_b32 s12, s12, 8
+; SI-NEXT: s_or_b32 s13, s13, s12
+; SI-NEXT: v_readlane_b32 s12, v56, 18
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_lshl_b32 s27, s55, 8
-; SI-NEXT: s_or_b32 s52, s12, s27
-; SI-NEXT: s_and_b32 s12, s53, 0xff
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s27, s54, 24
-; SI-NEXT: s_or_b32 s17, s27, s12
-; SI-NEXT: s_and_b32 s12, s67, 0xff
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s27, s66, 24
-; SI-NEXT: s_or_b32 s19, s27, s12
-; SI-NEXT: s_and_b32 s12, s64, 0xff
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s44, s65, 24
-; SI-NEXT: s_or_b32 s12, s44, s12
-; SI-NEXT: s_and_b32 s44, s71, 0xff
-; SI-NEXT: s_lshl_b32 s46, s70, 8
-; SI-NEXT: s_or_b32 s53, s44, s46
-; SI-NEXT: s_and_b32 s44, s68, 0xff
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s46, s69, 24
-; SI-NEXT: s_or_b32 s44, s46, s44
-; SI-NEXT: s_and_b32 s46, s9, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s41, s41, 24
-; SI-NEXT: s_or_b32 s41, s41, s46
-; SI-NEXT: s_and_b32 s46, s80, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s59, s8, 24
-; SI-NEXT: s_or_b32 s59, s59, s46
-; SI-NEXT: s_and_b32 s46, s11, 0xff
-; SI-NEXT: s_lshl_b32 s62, s29, 8
-; SI-NEXT: s_or_b32 s54, s46, s62
-; SI-NEXT: s_and_b32 s46, s24, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s62, s13, 24
-; SI-NEXT: s_or_b32 s27, s62, s46
-; SI-NEXT: s_and_b32 s46, s56, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s62, s40, 24
-; SI-NEXT: s_or_b32 s62, s62, s46
-; SI-NEXT: s_and_b32 s46, s14, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s75, s15, 24
-; SI-NEXT: s_or_b32 s75, s75, s46
-; SI-NEXT: s_and_b32 s46, s45, 0xff
-; SI-NEXT: s_lshl_b32 s77, s43, 8
-; SI-NEXT: s_or_b32 s55, s46, s77
-; SI-NEXT: s_and_b32 s46, s42, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s77, s58, 24
-; SI-NEXT: s_or_b32 s29, s77, s46
-; SI-NEXT: s_and_b32 s46, s72, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s79, s76, 24
-; SI-NEXT: s_or_b32 s46, s79, s46
-; SI-NEXT: s_and_b32 s79, s47, 0xff
+; SI-NEXT: s_lshl_b32 s14, s12, 16
+; SI-NEXT: v_readlane_b32 s12, v56, 17
+; SI-NEXT: s_lshl_b32 s12, s12, 24
+; SI-NEXT: s_or_b32 s86, s12, s14
+; SI-NEXT: v_readlane_b32 s12, v56, 22
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_lshl_b32 s14, s12, 16
+; SI-NEXT: v_readlane_b32 s12, v56, 21
+; SI-NEXT: s_lshl_b32 s12, s12, 24
+; SI-NEXT: s_or_b32 s12, s12, s14
+; SI-NEXT: v_readlane_b32 s14, v56, 12
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s15, s27, 8
+; SI-NEXT: s_or_b32 s15, s14, s15
+; SI-NEXT: s_and_b32 s14, s26, 0xff
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_lshl_b32 s16, vcc_hi, 24
+; SI-NEXT: s_or_b32 s29, s16, s14
+; SI-NEXT: v_readlane_b32 s14, v56, 14
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s16, s14, 16
+; SI-NEXT: v_readlane_b32 s14, v56, 13
+; SI-NEXT: s_lshl_b32 s14, s14, 24
+; SI-NEXT: s_or_b32 s14, s14, s16
+; SI-NEXT: s_and_b32 s16, s28, 0xff
+; SI-NEXT: s_lshl_b32 s18, s25, 8
+; SI-NEXT: s_or_b32 s18, s16, s18
+; SI-NEXT: s_and_b32 s16, s85, 0xff
+; SI-NEXT: v_writelane_b32 v46, s20, 1
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshl_b32 s20, s22, 24
+; SI-NEXT: v_writelane_b32 v46, vcc_hi, 2
+; SI-NEXT: v_writelane_b32 v46, s28, 3
+; SI-NEXT: s_or_b32 s28, s20, s16
+; SI-NEXT: s_and_b32 s20, s87, 0xff
+; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: v_writelane_b32 v46, s24, 4
+; SI-NEXT: s_lshl_b32 s24, s19, 24
+; SI-NEXT: s_and_b32 s18, s18, 0xffff
+; SI-NEXT: s_or_b32 s40, s24, s20
+; SI-NEXT: s_or_b32 s41, s18, s28
+; SI-NEXT: s_and_b32 s18, s79, 0xff
+; SI-NEXT: s_lshl_b32 s20, s75, 8
+; SI-NEXT: s_or_b32 s20, s18, s20
+; SI-NEXT: s_and_b32 s18, s42, 0xff
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_lshl_b32 s24, s43, 24
+; SI-NEXT: s_or_b32 s19, s24, s18
+; SI-NEXT: s_and_b32 s24, s89, 0xff
+; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: v_writelane_b32 v46, s25, 5
+; SI-NEXT: s_lshl_b32 s25, s91, 24
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s42, s25, s24
+; SI-NEXT: s_or_b32 s43, s20, s19
+; SI-NEXT: s_and_b32 s20, s31, 0xff
+; SI-NEXT: s_lshl_b32 s24, s95, 8
+; SI-NEXT: s_or_b32 s20, s20, s24
+; SI-NEXT: s_and_b32 s24, s92, 0xff
+; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: s_lshl_b32 s25, s93, 24
+; SI-NEXT: s_or_b32 s24, s25, s24
+; SI-NEXT: s_and_b32 s25, s34, 0xff
+; SI-NEXT: s_mov_b32 s21, s26
+; SI-NEXT: s_lshl_b32 s25, s25, 16
+; SI-NEXT: s_lshl_b32 s26, s36, 24
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s34, s26, s25
+; SI-NEXT: s_or_b32 s35, s20, s24
+; SI-NEXT: s_and_b32 s20, s49, 0xff
+; SI-NEXT: s_lshl_b32 s25, s48, 8
+; SI-NEXT: s_or_b32 s20, s20, s25
+; SI-NEXT: s_and_b32 s25, s38, 0xff
+; SI-NEXT: s_lshl_b32 s25, s25, 16
+; SI-NEXT: s_lshl_b32 s26, s39, 24
+; SI-NEXT: v_writelane_b32 v46, s27, 6
+; SI-NEXT: s_or_b32 s27, s26, s25
+; SI-NEXT: s_and_b32 s26, s50, 0xff
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s16, s51, 24
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s36, s16, s26
+; SI-NEXT: s_or_b32 s37, s20, s27
+; SI-NEXT: s_and_b32 s20, s88, 0xff
+; SI-NEXT: s_lshl_b32 s26, s67, 8
+; SI-NEXT: s_or_b32 s20, s20, s26
+; SI-NEXT: s_and_b32 s26, s52, 0xff
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s16, s53, 24
+; SI-NEXT: s_or_b32 s18, s16, s26
+; SI-NEXT: s_and_b32 s26, s78, 0xff
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s75, s66, 24
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s38, s75, s26
+; SI-NEXT: s_or_b32 s39, s20, s18
+; SI-NEXT: s_and_b32 s20, s63, 0xff
+; SI-NEXT: s_lshl_b32 s26, s61, 8
+; SI-NEXT: s_or_b32 s20, s20, s26
+; SI-NEXT: s_and_b32 s26, s58, 0xff
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s75, s59, 24
+; SI-NEXT: s_or_b32 s26, s75, s26
+; SI-NEXT: s_and_b32 s75, s74, 0xff
+; SI-NEXT: s_lshl_b32 s75, s75, 16
+; SI-NEXT: s_lshl_b32 s78, s64, 24
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_or_b32 s48, s78, s75
+; SI-NEXT: s_or_b32 s49, s20, s26
+; SI-NEXT: s_and_b32 s20, s23, 0xff
+; SI-NEXT: s_lshl_b32 s75, vcc_lo, 8
+; SI-NEXT: s_or_b32 s75, s20, s75
+; SI-NEXT: s_and_b32 s20, s60, 0xff
+; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_lshl_b32 s78, s56, 24
+; SI-NEXT: s_or_b32 s20, s78, s20
+; SI-NEXT: s_and_b32 s78, s94, 0xff
+; SI-NEXT: s_lshl_b32 s78, s78, 16
+; SI-NEXT: s_lshl_b32 s79, s65, 24
+; SI-NEXT: s_and_b32 s75, s75, 0xffff
+; SI-NEXT: s_or_b32 s50, s79, s78
+; SI-NEXT: s_or_b32 s51, s75, s20
+; SI-NEXT: s_and_b32 s75, s62, 0xff
+; SI-NEXT: s_lshl_b32 s78, s17, 8
+; SI-NEXT: s_or_b32 s75, s75, s78
+; SI-NEXT: s_and_b32 s78, s45, 0xff
+; SI-NEXT: s_lshl_b32 s78, s78, 16
+; SI-NEXT: s_lshl_b32 s79, s44, 24
+; SI-NEXT: s_or_b32 s78, s79, s78
+; SI-NEXT: s_and_b32 s79, s90, 0xff
; SI-NEXT: s_lshl_b32 s79, s79, 16
-; SI-NEXT: s_lshl_b32 s88, s60, 24
-; SI-NEXT: s_or_b32 s24, s88, s79
-; SI-NEXT: s_and_b32 s88, s81, 0xff
-; SI-NEXT: s_lshl_b32 s89, s57, 8
-; SI-NEXT: s_mov_b32 s42, s16
-; SI-NEXT: s_or_b32 s16, s88, s89
-; SI-NEXT: s_and_b32 s88, s91, 0xff
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_lshl_b32 s89, s21, 24
-; SI-NEXT: s_or_b32 s21, s89, s88
-; SI-NEXT: s_and_b32 s88, s90, 0xff
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_lshl_b32 s90, s99, 24
-; SI-NEXT: s_or_b32 s88, s90, s88
-; SI-NEXT: s_and_b32 s90, s87, 0xff
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: s_lshl_b32 s91, s96, 24
-; SI-NEXT: s_or_b32 s40, s91, s90
-; SI-NEXT: s_and_b32 s91, s73, 0xff
-; SI-NEXT: s_lshl_b32 s92, s25, 8
-; SI-NEXT: s_mov_b32 s45, s19
-; SI-NEXT: s_or_b32 s19, s91, s92
-; SI-NEXT: s_and_b32 s91, s28, 0xff
-; SI-NEXT: s_lshl_b32 s91, s91, 16
-; SI-NEXT: s_lshl_b32 s92, s26, 24
-; SI-NEXT: s_or_b32 s92, s92, s91
-; SI-NEXT: s_and_b32 s91, s94, 0xff
-; SI-NEXT: s_lshl_b32 s91, s91, 16
-; SI-NEXT: s_lshl_b32 s93, s93, 24
-; SI-NEXT: s_or_b32 s91, s93, s91
-; SI-NEXT: s_and_b32 s93, s74, 0xff
-; SI-NEXT: s_lshl_b32 s93, s93, 16
-; SI-NEXT: s_lshl_b32 s94, s97, 24
-; SI-NEXT: s_or_b32 s93, s94, s93
-; SI-NEXT: s_and_b32 s94, s98, 0xff
-; SI-NEXT: s_lshl_b32 s95, s95, 8
-; SI-NEXT: s_or_b32 s22, s94, s95
-; SI-NEXT: s_and_b32 s94, s23, 0xff
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_lshl_b32 s95, s50, 24
-; SI-NEXT: s_or_b32 s95, s95, s94
-; SI-NEXT: s_and_b32 s94, s34, 0xff
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_lshl_b32 vcc_lo, s49, 24
-; SI-NEXT: s_or_b32 s94, vcc_lo, s94
-; SI-NEXT: s_and_b32 vcc_lo, s61, 0xff
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_lshl_b32 vcc_hi, s86, 24
-; SI-NEXT: s_or_b32 vcc_lo, vcc_hi, vcc_lo
-; SI-NEXT: s_and_b32 vcc_hi, s84, 0xff
-; SI-NEXT: s_lshl_b32 s30, s30, 8
-; SI-NEXT: s_mov_b32 s43, s17
-; SI-NEXT: s_or_b32 s17, vcc_hi, s30
-; SI-NEXT: s_and_b32 vcc_hi, s38, 0xff
-; SI-NEXT: s_lshl_b32 vcc_hi, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s30, s63, 24
-; SI-NEXT: s_or_b32 s30, s30, vcc_hi
-; SI-NEXT: s_and_b32 vcc_hi, s48, 0xff
-; SI-NEXT: s_lshl_b32 vcc_hi, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s31, s31, 24
-; SI-NEXT: s_or_b32 vcc_hi, s31, vcc_hi
-; SI-NEXT: s_and_b32 s31, s39, 0xff
-; SI-NEXT: s_lshl_b32 s31, s31, 16
-; SI-NEXT: s_lshl_b32 s34, s35, 24
-; SI-NEXT: s_or_b32 s31, s34, s31
-; SI-NEXT: s_and_b32 s34, s82, 0xff
-; SI-NEXT: s_lshl_b32 s35, s85, 8
-; SI-NEXT: s_or_b32 s20, s34, s35
-; SI-NEXT: s_and_b32 s34, s78, 0xff
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: s_lshl_b32 s35, s37, 24
-; SI-NEXT: v_readlane_b32 s8, v56, 31
-; SI-NEXT: s_or_b32 s35, s35, s34
-; SI-NEXT: s_and_b32 s34, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v56, 30
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: s_lshl_b32 s36, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v56, 29
-; SI-NEXT: s_or_b32 s34, s36, s34
-; SI-NEXT: s_and_b32 s36, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v56, 28
-; SI-NEXT: s_lshl_b32 s36, s36, 16
-; SI-NEXT: s_lshl_b32 s37, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v56, 27
-; SI-NEXT: s_or_b32 s36, s37, s36
-; SI-NEXT: s_and_b32 s37, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v56, 26
-; SI-NEXT: s_lshl_b32 s38, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v56, 25
-; SI-NEXT: s_or_b32 s23, s37, s38
-; SI-NEXT: s_and_b32 s37, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v56, 24
-; SI-NEXT: s_lshl_b32 s37, s37, 16
-; SI-NEXT: s_lshl_b32 s38, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v56, 23
-; SI-NEXT: s_or_b32 s38, s38, s37
-; SI-NEXT: s_mov_b32 s90, s8
-; SI-NEXT: s_and_b32 s37, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v56, 22
-; SI-NEXT: s_lshl_b32 s37, s37, 16
-; SI-NEXT: s_mov_b32 s28, s8
-; SI-NEXT: s_lshl_b32 s39, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v56, 21
-; SI-NEXT: s_or_b32 s37, s39, s37
-; SI-NEXT: s_and_b32 s39, s8, 0xff
-; SI-NEXT: s_lshl_b32 s39, s39, 16
-; SI-NEXT: s_lshl_b32 s48, s10, 24
-; SI-NEXT: s_or_b32 s39, s48, s39
-; SI-NEXT: s_and_b32 s48, s51, 0xff
-; SI-NEXT: s_lshl_b32 s64, s83, 8
-; SI-NEXT: v_readlane_b32 s8, v46, 8
-; SI-NEXT: s_or_b32 s25, s48, s64
-; SI-NEXT: s_mov_b32 s57, s8
-; SI-NEXT: s_and_b32 s48, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v46, 7
-; SI-NEXT: s_mov_b32 s76, s8
-; SI-NEXT: s_lshl_b32 s64, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v46, 9
-; SI-NEXT: s_lshl_b32 s80, s8, 16
-; SI-NEXT: v_readlane_b32 s8, v46, 10
-; SI-NEXT: s_lshl_b32 s71, s8, 16
-; SI-NEXT: v_readlane_b32 s8, v46, 11
-; SI-NEXT: s_lshl_b32 s48, s48, 16
-; SI-NEXT: s_lshl_b32 s70, s8, 16
-; SI-NEXT: v_readlane_b32 s8, v46, 12
-; SI-NEXT: s_mov_b32 s74, s10
-; SI-NEXT: s_mov_b32 s26, s51
-; SI-NEXT: s_mov_b32 s73, s83
-; SI-NEXT: s_or_b32 s48, s64, s48
-; SI-NEXT: s_lshl_b32 s69, s8, 16
-; SI-NEXT: s_lshl_b32 s68, s6, 16
-; SI-NEXT: s_lshl_b32 s67, s52, 16
-; SI-NEXT: s_lshl_b32 s66, s53, 16
-; SI-NEXT: s_lshl_b32 s65, s54, 16
-; SI-NEXT: s_lshl_b32 s64, s55, 16
-; SI-NEXT: s_lshl_b32 s55, s16, 16
-; SI-NEXT: s_lshl_b32 s54, s19, 16
-; SI-NEXT: s_mov_b32 s19, s5
-; SI-NEXT: s_lshl_b32 s53, s22, 16
-; SI-NEXT: s_lshl_b32 s52, s17, 16
-; SI-NEXT: s_lshl_b32 s51, s20, 16
-; SI-NEXT: s_mov_b32 s20, s4
-; SI-NEXT: s_lshl_b32 s50, s23, 16
-; SI-NEXT: s_lshl_b32 s49, s25, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshl_b32 s88, s84, 24
+; SI-NEXT: s_and_b32 s75, s75, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 47
+; SI-NEXT: s_or_b32 s52, s88, s79
+; SI-NEXT: s_or_b32 s53, s75, s78
+; SI-NEXT: s_and_b32 s75, s46, 0xff
+; SI-NEXT: s_lshl_b32 s79, s16, 8
+; SI-NEXT: s_or_b32 s75, s75, s79
+; SI-NEXT: s_and_b32 s79, s55, 0xff
+; SI-NEXT: s_lshl_b32 s79, s79, 16
+; SI-NEXT: s_lshl_b32 s88, s54, 24
+; SI-NEXT: s_or_b32 s88, s88, s79
+; SI-NEXT: s_and_b32 s79, s73, 0xff
+; SI-NEXT: s_lshl_b32 s79, s79, 16
+; SI-NEXT: s_lshl_b32 s89, s72, 24
+; SI-NEXT: s_and_b32 s75, s75, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 48
+; SI-NEXT: s_or_b32 s54, s89, s79
+; SI-NEXT: s_or_b32 s55, s75, s88
+; SI-NEXT: s_and_b32 s75, s83, 0xff
+; SI-NEXT: s_lshl_b32 s79, s82, 8
+; SI-NEXT: s_and_b32 s44, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 49
+; SI-NEXT: s_or_b32 s75, s75, s79
+; SI-NEXT: s_and_b32 s79, s81, 0xff
+; SI-NEXT: s_and_b32 s45, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 50
+; SI-NEXT: s_lshl_b32 s79, s79, 16
+; SI-NEXT: s_lshl_b32 s89, s80, 24
+; SI-NEXT: s_and_b32 s46, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 51
+; SI-NEXT: s_or_b32 s90, s89, s79
+; SI-NEXT: s_lshl_b32 s89, s47, 24
+; SI-NEXT: s_and_b32 s47, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 52
+; SI-NEXT: s_and_b32 s79, s57, 0xff
+; SI-NEXT: s_and_b32 s56, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 53
+; SI-NEXT: s_lshl_b32 s79, s79, 16
+; SI-NEXT: s_and_b32 s57, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 54
+; SI-NEXT: s_or_b32 s64, s89, s79
+; SI-NEXT: s_and_b32 s79, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 55
+; SI-NEXT: s_and_b32 s89, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 56
+; SI-NEXT: s_and_b32 s17, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 57
+; SI-NEXT: s_and_b32 s91, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 58
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_and_b32 s92, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 59
+; SI-NEXT: s_or_b32 s13, s13, s86
+; SI-NEXT: s_and_b32 s93, s16, 0xffff
+; SI-NEXT: s_mov_b32 s16, s85
+; SI-NEXT: s_lshr_b64 s[84:85], s[40:41], 16
+; SI-NEXT: s_and_b32 s94, s71, 0xffff
+; SI-NEXT: s_and_b32 s95, s70, 0xffff
+; SI-NEXT: s_or_b32 s58, s56, s12
+; SI-NEXT: s_mov_b32 s59, s13
+; SI-NEXT: s_lshr_b64 s[70:71], s[12:13], 16
+; SI-NEXT: s_or_b32 s12, s93, s48
+; SI-NEXT: s_mov_b32 s13, s49
+; SI-NEXT: s_lshr_b64 s[48:49], s[48:49], 16
+; SI-NEXT: s_mov_b32 s85, s16
+; SI-NEXT: v_readlane_b32 s16, v47, 61
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_mov_b32 s25, s22
+; SI-NEXT: s_and_b32 s75, s75, 0xffff
+; SI-NEXT: v_readlane_b32 s22, v47, 60
+; SI-NEXT: s_or_b32 s60, s47, s10
+; SI-NEXT: s_mov_b32 s61, s11
+; SI-NEXT: s_lshr_b64 vcc, s[10:11], 16
+; SI-NEXT: s_or_b32 s10, s94, s50
+; SI-NEXT: s_mov_b32 s11, s51
+; SI-NEXT: s_lshr_b64 s[50:51], s[50:51], 16
+; SI-NEXT: s_lshr_b32 s49, s16, 16
+; SI-NEXT: v_readlane_b32 s16, v47, 62
+; SI-NEXT: s_or_b32 s15, s15, s29
+; SI-NEXT: s_mov_b32 s23, s21
+; SI-NEXT: s_or_b32 s65, s75, s90
+; SI-NEXT: s_and_b32 s21, s69, 0xffff
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s62, s46, s8
+; SI-NEXT: s_mov_b32 s63, s9
+; SI-NEXT: s_lshr_b64 s[66:67], s[8:9], 16
+; SI-NEXT: s_or_b32 s8, s95, s52
+; SI-NEXT: s_mov_b32 s9, s53
+; SI-NEXT: s_lshr_b64 s[52:53], s[52:53], 16
+; SI-NEXT: s_lshr_b32 s51, s16, 16
+; SI-NEXT: v_readlane_b32 s16, v47, 63
+; SI-NEXT: s_or_b32 s74, s44, s4
+; SI-NEXT: s_mov_b32 s75, s5
+; SI-NEXT: s_lshr_b64 s[82:83], s[4:5], 16
+; SI-NEXT: s_or_b32 s72, s45, s6
+; SI-NEXT: s_mov_b32 s73, s7
+; SI-NEXT: s_lshr_b64 s[30:31], s[6:7], 16
+; SI-NEXT: s_or_b32 s56, s57, s14
+; SI-NEXT: s_mov_b32 s57, s15
+; SI-NEXT: s_lshr_b64 s[80:81], s[14:15], 16
+; SI-NEXT: s_or_b32 s44, s89, s42
+; SI-NEXT: s_mov_b32 s45, s43
+; SI-NEXT: s_lshr_b64 s[68:69], s[42:43], 16
+; SI-NEXT: s_or_b32 s42, s17, s34
+; SI-NEXT: s_mov_b32 s43, s35
+; SI-NEXT: s_lshr_b64 s[34:35], s[34:35], 16
+; SI-NEXT: s_or_b32 s14, s92, s38
+; SI-NEXT: s_mov_b32 s15, s39
+; SI-NEXT: s_lshr_b64 s[38:39], s[38:39], 16
+; SI-NEXT: s_or_b32 s6, s21, s54
+; SI-NEXT: s_mov_b32 s7, s55
+; SI-NEXT: s_lshr_b64 s[54:55], s[54:55], 16
+; SI-NEXT: s_or_b32 s4, s22, s64
+; SI-NEXT: s_mov_b32 s5, s65
+; SI-NEXT: s_lshr_b64 s[64:65], s[64:65], 16
+; SI-NEXT: s_lshr_b32 s53, s16, 16
+; SI-NEXT: v_readlane_b32 s16, v46, 0
+; SI-NEXT: s_or_b32 s46, s79, s40
+; SI-NEXT: s_mov_b32 s47, s41
+; SI-NEXT: s_or_b32 s40, s91, s36
+; SI-NEXT: s_mov_b32 s41, s37
+; SI-NEXT: s_lshr_b64 s[36:37], s[36:37], 16
+; SI-NEXT: s_lshr_b32 s55, s16, 16
+; SI-NEXT: s_lshr_b32 s65, s86, 16
+; SI-NEXT: s_lshr_b32 s35, s29, 16
+; SI-NEXT: s_mov_b32 s94, s66
+; SI-NEXT: s_lshr_b32 s79, s28, 16
+; SI-NEXT: v_readlane_b32 s28, v46, 3
+; SI-NEXT: v_readlane_b32 vcc_hi, v46, 2
+; SI-NEXT: s_lshr_b32 s89, s19, 16
+; SI-NEXT: s_lshr_b32 s91, s24, 16
+; SI-NEXT: v_readlane_b32 s24, v46, 4
+; SI-NEXT: s_lshr_b32 s93, s27, 16
+; SI-NEXT: v_readlane_b32 s27, v46, 6
+; SI-NEXT: s_lshr_b32 s95, s18, 16
+; SI-NEXT: s_lshr_b32 s31, s26, 16
+; SI-NEXT: s_lshr_b32 s17, s20, 16
+; SI-NEXT: v_readlane_b32 s20, v46, 1
+; SI-NEXT: s_lshr_b32 s92, s78, 16
+; SI-NEXT: s_lshr_b32 s78, s88, 16
+; SI-NEXT: v_readlane_b32 s21, v47, 47
+; SI-NEXT: s_mov_b32 s26, s23
+; SI-NEXT: s_mov_b32 s22, s25
+; SI-NEXT: v_readlane_b32 s25, v46, 5
+; SI-NEXT: s_lshr_b32 s39, s90, 16
+; SI-NEXT: s_mov_b64 s[66:67], 0
; SI-NEXT: s_branch .LBB89_3
; SI-NEXT: .LBB89_2:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: v_readlane_b32 s90, v56, 23
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: v_readlane_b32 s28, v56, 22
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b32 s26, s10
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b32 s74, s5
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b32 s73, s51
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b32 s57, s4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b32 s76, s83
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: s_mov_b64 s[66:67], -1
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr51
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr95
; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr42
; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr17
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr92
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr39
; SI-NEXT: .LBB89_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: v_readlane_b32 s81, v47, 59
-; SI-NEXT: v_readlane_b32 s82, v47, 60
-; SI-NEXT: v_readlane_b32 s84, v47, 61
-; SI-NEXT: v_readlane_b32 s83, v47, 62
-; SI-NEXT: v_readlane_b32 s85, v47, 63
-; SI-NEXT: v_readlane_b32 s87, v46, 0
-; SI-NEXT: v_readlane_b32 s86, v46, 1
-; SI-NEXT: v_readlane_b32 s96, v46, 2
-; SI-NEXT: v_readlane_b32 s98, v46, 3
-; SI-NEXT: v_readlane_b32 s97, v46, 4
-; SI-NEXT: v_readlane_b32 s99, v46, 5
-; SI-NEXT: v_readlane_b32 s6, v46, 6
+; SI-NEXT: s_and_b64 s[18:19], s[66:67], exec
+; SI-NEXT: s_cselect_b32 s16, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s16, 1
; SI-NEXT: s_cbranch_scc1 .LBB89_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s4, s26, 3
+; SI-NEXT: v_readlane_b32 s4, v47, 32
+; SI-NEXT: s_add_i32 s4, s4, 3
+; SI-NEXT: v_readlane_b32 s5, v47, 27
+; SI-NEXT: v_readlane_b32 s6, v47, 17
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s73, 8
-; SI-NEXT: s_add_i32 s6, s57, 3
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_readlane_b32 s5, v47, 15
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s5, s76, 24
+; SI-NEXT: s_lshl_b32 s5, s5, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_addk_i32 s4, 0x300
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s5, s90, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 21
+; SI-NEXT: v_readlane_b32 s5, v47, 2
+; SI-NEXT: s_add_i32 s5, s5, 3
+; SI-NEXT: v_readlane_b32 s6, v56, 62
+; SI-NEXT: v_readlane_b32 s7, v56, 55
; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: s_lshl_b32 s6, s28, 8
+; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: v_readlane_b32 s6, v56, 52
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s6, s74, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_addk_i32 s5, 0x300
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s6, v56, 27
-; SI-NEXT: s_add_i32 s21, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 26
-; SI-NEXT: v_readlane_b32 s8, v56, 25
-; SI-NEXT: s_and_b32 s6, s21, 0xff
+; SI-NEXT: v_readlane_b32 s6, v47, 41
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 35
+; SI-NEXT: v_readlane_b32 s8, v47, 30
+; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s8, 3
+; SI-NEXT: s_add_i32 s8, s8, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 24
-; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: v_readlane_b32 s7, v47, 29
+; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: s_or_b32 s7, s7, s8
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 31
-; SI-NEXT: s_add_i32 s10, s7, 3
-; SI-NEXT: v_readlane_b32 s8, v56, 30
-; SI-NEXT: s_and_b32 s7, s10, 0xff
-; SI-NEXT: s_lshl_b32 s10, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v56, 28
-; SI-NEXT: s_or_b32 s7, s10, s7
-; SI-NEXT: s_lshl_b32 s10, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v56, 29
-; SI-NEXT: s_add_i32 s12, s8, 3
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: v_readlane_b32 s7, v47, 12
+; SI-NEXT: s_add_i32 s7, s7, 3
+; SI-NEXT: v_readlane_b32 s9, v47, 6
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s8, s21, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: v_readlane_b32 s8, v47, 1
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: v_readlane_b32 s8, v47, 50
-; SI-NEXT: s_or_b32 s10, s10, s12
+; SI-NEXT: s_or_b32 s8, s8, s9
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_add_i32 s57, s8, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 41
-; SI-NEXT: s_or_b32 s7, s10, s7
-; SI-NEXT: s_and_b32 s10, s57, 0xff
-; SI-NEXT: s_lshl_b32 s12, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v47, 33
-; SI-NEXT: s_or_b32 s10, s12, s10
-; SI-NEXT: s_lshl_b32 s12, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v47, 35
-; SI-NEXT: s_add_i32 s26, s8, 3
-; SI-NEXT: s_and_b32 s13, s26, 0xff
-; SI-NEXT: v_readlane_b32 s8, v47, 55
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_add_i32 s73, s8, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 51
-; SI-NEXT: s_or_b32 s12, s12, s13
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: v_readlane_b32 s8, v47, 44
+; SI-NEXT: s_add_i32 s8, s8, 3
+; SI-NEXT: v_readlane_b32 s9, v47, 43
+; SI-NEXT: v_readlane_b32 s10, v47, 40
+; SI-NEXT: s_and_b32 s8, s8, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 8
+; SI-NEXT: s_add_i32 s10, s10, 3
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: v_readlane_b32 s9, v47, 37
+; SI-NEXT: s_and_b32 s10, s10, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_addk_i32 s8, 0x300
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: v_readlane_b32 s9, v47, 25
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: v_readlane_b32 s10, v47, 21
+; SI-NEXT: v_readlane_b32 s11, v47, 11
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_lshl_b32 s10, s10, 8
+; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: v_readlane_b32 s10, v47, 10
+; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: s_lshl_b32 s10, s10, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_addk_i32 s9, 0x300
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: v_readlane_b32 s10, v47, 46
+; SI-NEXT: s_add_i32 s10, s10, 3
+; SI-NEXT: v_readlane_b32 s11, v47, 45
+; SI-NEXT: v_readlane_b32 s12, v47, 42
+; SI-NEXT: s_and_b32 s10, s10, 0xff
+; SI-NEXT: s_lshl_b32 s11, s11, 8
+; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: v_readlane_b32 s11, v47, 39
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_lshl_b32 s11, s11, 24
+; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s13, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v47, 39
+; SI-NEXT: s_or_b32 s11, s11, s12
; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v47, 40
-; SI-NEXT: s_or_b32 s10, s12, s10
-; SI-NEXT: s_and_b32 s12, s73, 0xff
-; SI-NEXT: s_add_i32 s11, s8, 3
-; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: v_readlane_b32 s11, v47, 31
+; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: v_readlane_b32 s12, v47, 28
+; SI-NEXT: v_readlane_b32 s13, v47, 22
; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_addk_i32 s12, 0x300
-; SI-NEXT: v_readlane_b32 s8, v47, 56
-; SI-NEXT: s_or_b32 s9, s9, s11
-; SI-NEXT: s_and_b32 s11, s12, 0xffff
-; SI-NEXT: s_add_i32 s74, s8, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 53
-; SI-NEXT: s_or_b32 s9, s9, s11
-; SI-NEXT: s_and_b32 s11, s74, 0xff
-; SI-NEXT: s_lshl_b32 s12, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v47, 43
+; SI-NEXT: s_lshl_b32 s12, s12, 8
+; SI-NEXT: s_add_i32 s13, s13, 3
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_lshl_b32 s12, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v47, 46
-; SI-NEXT: s_add_i32 s43, s8, 3
-; SI-NEXT: s_and_b32 s13, s43, 0xff
+; SI-NEXT: v_readlane_b32 s12, v47, 16
+; SI-NEXT: s_and_b32 s13, s13, 0xff
+; SI-NEXT: s_lshl_b32 s12, s12, 24
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_addk_i32 s11, 0x300
-; SI-NEXT: v_readlane_b32 s8, v47, 58
; SI-NEXT: s_or_b32 s12, s12, s13
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_add_i32 s78, s8, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 57
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s78, 0xff
-; SI-NEXT: s_lshl_b32 s13, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v47, 44
+; SI-NEXT: v_readlane_b32 s12, v47, 38
+; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: v_readlane_b32 s13, v47, 36
+; SI-NEXT: v_readlane_b32 s14, v47, 33
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_lshl_b32 s13, s13, 8
+; SI-NEXT: s_add_i32 s14, s14, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_lshl_b32 s13, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v47, 47
-; SI-NEXT: s_add_i32 s45, s8, 3
-; SI-NEXT: s_and_b32 s15, s45, 0xff
-; SI-NEXT: s_lshl_b32 s15, s15, 16
+; SI-NEXT: v_readlane_b32 s13, v47, 34
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s13, s13, 24
+; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_addk_i32 s12, 0x300
-; SI-NEXT: v_readlane_b32 s8, v47, 54
-; SI-NEXT: s_or_b32 s13, s13, s15
+; SI-NEXT: s_or_b32 s13, s13, s14
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_add_i32 s72, s8, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 52
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s72, 0xff
-; SI-NEXT: s_lshl_b32 s15, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v47, 49
-; SI-NEXT: s_or_b32 s13, s15, s13
-; SI-NEXT: s_lshl_b32 s15, s8, 24
-; SI-NEXT: v_readlane_b32 s8, v47, 48
-; SI-NEXT: s_add_i32 s47, s8, 3
-; SI-NEXT: s_and_b32 s18, s47, 0xff
-; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: v_readlane_b32 s13, v47, 26
+; SI-NEXT: s_add_i32 s13, s13, 3
+; SI-NEXT: v_readlane_b32 s14, v47, 24
+; SI-NEXT: v_readlane_b32 s15, v47, 19
+; SI-NEXT: s_and_b32 s13, s13, 0xff
+; SI-NEXT: s_lshl_b32 s14, s14, 8
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: v_readlane_b32 s14, v47, 20
+; SI-NEXT: s_and_b32 s15, s15, 0xff
+; SI-NEXT: s_lshl_b32 s14, s14, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 16
; SI-NEXT: s_addk_i32 s13, 0x300
-; SI-NEXT: v_readlane_b32 s8, v47, 45
-; SI-NEXT: s_or_b32 s15, s15, s18
-; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_add_i32 s42, s8, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 42
-; SI-NEXT: s_or_b32 s13, s15, s13
-; SI-NEXT: s_and_b32 s15, s42, 0xff
-; SI-NEXT: s_lshl_b32 s14, s8, 8
; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: v_readlane_b32 s15, v47, 37
-; SI-NEXT: s_add_i32 s29, s15, 3
-; SI-NEXT: v_readlane_b32 s8, v47, 38
-; SI-NEXT: s_and_b32 s15, s29, 0xff
-; SI-NEXT: s_lshl_b32 s8, s8, 24
-; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_addk_i32 s14, 0x300
-; SI-NEXT: s_or_b32 s8, s8, s15
-; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_or_b32 s14, s8, s14
-; SI-NEXT: v_readlane_b32 s8, v47, 36
-; SI-NEXT: s_add_i32 s24, s8, 3
-; SI-NEXT: v_readlane_b32 s15, v47, 34
-; SI-NEXT: v_readlane_b32 s16, v47, 31
-; SI-NEXT: s_and_b32 s8, s24, 0xff
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: v_readlane_b32 s14, v47, 23
+; SI-NEXT: s_add_i32 s14, s14, 3
+; SI-NEXT: v_readlane_b32 s15, v47, 18
+; SI-NEXT: v_readlane_b32 s16, v47, 13
+; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 8
-; SI-NEXT: s_add_i32 s17, s16, 3
-; SI-NEXT: s_or_b32 s8, s15, s8
-; SI-NEXT: v_readlane_b32 s15, v47, 32
-; SI-NEXT: s_and_b32 s16, s17, 0xff
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: v_readlane_b32 s15, v47, 14
+; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
+; SI-NEXT: s_addk_i32 s14, 0x300
; SI-NEXT: s_or_b32 s15, s15, s16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s15, s15, s8
-; SI-NEXT: v_readlane_b32 s8, v47, 30
-; SI-NEXT: s_add_i32 s8, s8, 3
-; SI-NEXT: v_readlane_b32 s16, v47, 29
-; SI-NEXT: v_readlane_b32 s17, v47, 27
-; SI-NEXT: s_and_b32 s8, s8, 0xff
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: v_readlane_b32 s15, v47, 9
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: v_readlane_b32 s16, v47, 8
+; SI-NEXT: v_readlane_b32 s17, v47, 4
+; SI-NEXT: s_and_b32 s15, s15, 0xff
; SI-NEXT: s_lshl_b32 s16, s16, 8
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s8, s16, s8
-; SI-NEXT: v_readlane_b32 s16, v47, 28
+; SI-NEXT: s_or_b32 s15, s16, s15
+; SI-NEXT: v_readlane_b32 s16, v47, 5
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: s_lshl_b32 s16, s16, 24
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
+; SI-NEXT: s_addk_i32 s15, 0x300
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_add_i32 s23, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v47, 26
-; SI-NEXT: s_or_b32 s16, s16, s8
-; SI-NEXT: s_add_i32 s22, s7, 0x3000000
-; SI-NEXT: s_add_i32 s8, s10, 0x3000000
-; SI-NEXT: s_add_i32 s10, s11, 0x3000000
-; SI-NEXT: s_add_i32 s11, s12, 0x3000000
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v47, 25
-; SI-NEXT: v_readlane_b32 s12, v47, 23
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 24
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s17, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 22
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v47, 21
-; SI-NEXT: v_readlane_b32 s12, v47, 19
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 20
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s21, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 18
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v47, 17
-; SI-NEXT: v_readlane_b32 s12, v47, 15
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 16
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s24, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 14
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v47, 13
-; SI-NEXT: v_readlane_b32 s12, v47, 11
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 12
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s25, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 10
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v47, 9
-; SI-NEXT: v_readlane_b32 s12, v47, 7
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 8
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s26, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 6
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: s_and_b32 s7, s6, 0xff
-; SI-NEXT: v_readlane_b32 s6, v56, 32
-; SI-NEXT: v_readlane_b32 s12, v47, 4
-; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: v_readlane_b32 s7, v47, 5
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s29, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 3
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v47, 2
-; SI-NEXT: v_readlane_b32 s12, v47, 0
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 1
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s40, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v56, 63
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 62
-; SI-NEXT: v_readlane_b32 s12, v56, 60
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 61
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s27, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v56, 9
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 59
-; SI-NEXT: v_readlane_b32 s12, v56, 57
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 58
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s12, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v56, 13
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 12
-; SI-NEXT: v_readlane_b32 s18, v56, 11
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_or_b32 s15, s16, s15
+; SI-NEXT: v_readlane_b32 s16, v47, 7
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 3
+; SI-NEXT: v_readlane_b32 s18, v56, 63
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 10
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v47, 0
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s18
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s28, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v56, 17
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 16
-; SI-NEXT: v_readlane_b32 s18, v56, 15
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s40, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 61
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 60
+; SI-NEXT: v_readlane_b32 s18, v56, 57
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 14
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 58
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s18
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s20, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v56, 47
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v56, 20
-; SI-NEXT: v_readlane_b32 s18, v56, 19
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s41, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 59
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 56
+; SI-NEXT: v_readlane_b32 s18, v56, 53
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v56, 18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 54
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s18
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s7, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v56, 49
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s18, v56, 45
-; SI-NEXT: v_readlane_b32 s19, v56, 39
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s18, s18, 8
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_or_b32 s6, s18, s6
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s42, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 51
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 50
+; SI-NEXT: v_readlane_b32 s18, v56, 46
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 47
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s43, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 49
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 48
+; SI-NEXT: v_readlane_b32 s18, v56, 44
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 45
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s44, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 43
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 42
; SI-NEXT: v_readlane_b32 s18, v56, 38
-; SI-NEXT: s_and_b32 s19, s19, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s18, s18, s19
-; SI-NEXT: s_or_b32 s6, s18, s6
-; SI-NEXT: s_add_i32 s18, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v56, 51
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s19, v56, 48
-; SI-NEXT: v_readlane_b32 s41, v56, 41
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s19, s19, 8
-; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: s_or_b32 s6, s19, s6
-; SI-NEXT: v_readlane_b32 s19, v56, 40
-; SI-NEXT: s_and_b32 s41, s41, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s19, s19, 24
-; SI-NEXT: s_lshl_b32 s41, s41, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s19, s19, s41
-; SI-NEXT: s_or_b32 s6, s19, s6
-; SI-NEXT: v_readlane_b32 s19, v56, 52
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: v_readlane_b32 s41, v56, 50
-; SI-NEXT: v_readlane_b32 s42, v56, 43
-; SI-NEXT: s_and_b32 s19, s19, 0xff
-; SI-NEXT: s_lshl_b32 s41, s41, 8
-; SI-NEXT: s_add_i32 s42, s42, 3
-; SI-NEXT: s_or_b32 s19, s41, s19
-; SI-NEXT: v_readlane_b32 s41, v56, 42
-; SI-NEXT: s_and_b32 s42, s42, 0xff
-; SI-NEXT: s_addk_i32 s19, 0x300
-; SI-NEXT: s_lshl_b32 s41, s41, 24
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_or_b32 s41, s41, s42
-; SI-NEXT: s_or_b32 s19, s41, s19
-; SI-NEXT: v_readlane_b32 s41, v56, 55
-; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: v_readlane_b32 s42, v56, 53
-; SI-NEXT: v_readlane_b32 s43, v56, 46
-; SI-NEXT: s_and_b32 s41, s41, 0xff
-; SI-NEXT: s_lshl_b32 s42, s42, 8
-; SI-NEXT: s_add_i32 s43, s43, 3
-; SI-NEXT: s_or_b32 s41, s42, s41
-; SI-NEXT: v_readlane_b32 s42, v56, 44
-; SI-NEXT: s_and_b32 s43, s43, 0xff
-; SI-NEXT: s_addk_i32 s41, 0x300
-; SI-NEXT: s_lshl_b32 s42, s42, 24
-; SI-NEXT: s_lshl_b32 s43, s43, 16
-; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_or_b32 s42, s42, s43
-; SI-NEXT: s_or_b32 s41, s42, s41
-; SI-NEXT: v_readlane_b32 s42, v56, 1
-; SI-NEXT: s_add_i32 s42, s42, 3
-; SI-NEXT: v_readlane_b32 s43, v56, 0
-; SI-NEXT: v_readlane_b32 s44, v56, 56
-; SI-NEXT: s_and_b32 s42, s42, 0xff
-; SI-NEXT: s_lshl_b32 s43, s43, 8
-; SI-NEXT: s_add_i32 s44, s44, 3
-; SI-NEXT: s_or_b32 s42, s43, s42
-; SI-NEXT: v_readlane_b32 s43, v56, 54
-; SI-NEXT: s_and_b32 s44, s44, 0xff
-; SI-NEXT: s_addk_i32 s42, 0x300
-; SI-NEXT: s_lshl_b32 s43, s43, 24
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s42, s42, 0xffff
-; SI-NEXT: s_or_b32 s43, s43, s44
-; SI-NEXT: s_or_b32 s42, s43, s42
-; SI-NEXT: v_readlane_b32 s43, v56, 5
-; SI-NEXT: s_add_i32 s43, s43, 3
-; SI-NEXT: v_readlane_b32 s44, v56, 4
-; SI-NEXT: v_readlane_b32 s45, v56, 3
-; SI-NEXT: s_and_b32 s43, s43, 0xff
-; SI-NEXT: s_lshl_b32 s44, s44, 8
-; SI-NEXT: s_add_i32 s45, s45, 3
-; SI-NEXT: s_or_b32 s43, s44, s43
-; SI-NEXT: v_readlane_b32 s44, v56, 2
-; SI-NEXT: s_and_b32 s45, s45, 0xff
-; SI-NEXT: s_addk_i32 s43, 0x300
-; SI-NEXT: s_lshl_b32 s44, s44, 24
-; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_or_b32 s43, s44, s43
-; SI-NEXT: v_readlane_b32 s44, v56, 37
-; SI-NEXT: s_add_i32 s44, s44, 3
-; SI-NEXT: v_readlane_b32 s45, v56, 8
-; SI-NEXT: v_readlane_b32 s46, v56, 7
-; SI-NEXT: s_and_b32 s44, s44, 0xff
-; SI-NEXT: s_lshl_b32 s45, s45, 8
-; SI-NEXT: s_add_i32 s46, s46, 3
-; SI-NEXT: s_or_b32 s44, s45, s44
-; SI-NEXT: v_readlane_b32 s45, v56, 6
-; SI-NEXT: s_and_b32 s46, s46, 0xff
-; SI-NEXT: s_addk_i32 s44, 0x300
-; SI-NEXT: s_lshl_b32 s45, s45, 24
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_or_b32 s45, s45, s46
-; SI-NEXT: s_or_b32 s44, s45, s44
-; SI-NEXT: v_readlane_b32 s45, v56, 36
-; SI-NEXT: s_add_i32 s45, s45, 3
-; SI-NEXT: v_readlane_b32 s46, v56, 35
-; SI-NEXT: v_readlane_b32 s47, v56, 33
-; SI-NEXT: s_and_b32 s45, s45, 0xff
-; SI-NEXT: s_lshl_b32 s46, s46, 8
-; SI-NEXT: s_add_i32 s47, s47, 3
-; SI-NEXT: s_or_b32 s45, s46, s45
-; SI-NEXT: v_readlane_b32 s46, v56, 34
-; SI-NEXT: s_and_b32 s47, s47, 0xff
-; SI-NEXT: s_addk_i32 s45, 0x300
-; SI-NEXT: s_lshl_b32 s46, s46, 24
-; SI-NEXT: s_lshl_b32 s47, s47, 16
-; SI-NEXT: s_and_b32 s45, s45, 0xffff
-; SI-NEXT: s_or_b32 s46, s46, s47
-; SI-NEXT: s_or_b32 s45, s46, s45
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 39
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s45, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 41
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 40
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s87, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 37
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s46, s16, 0x3000000
+; SI-NEXT: s_add_i32 s16, s28, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s25, 8
+; SI-NEXT: s_add_i32 s18, s85, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s22, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s47, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 15
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_and_b32 s17, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v56, 13
+; SI-NEXT: s_lshl_b32 s18, s16, 24
+; SI-NEXT: v_readlane_b32 s16, v56, 14
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_or_b32 s16, s18, s16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_add_i32 s56, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 12
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s27, 8
+; SI-NEXT: s_add_i32 s18, s26, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, vcc_hi, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s57, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 24
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s17, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v56, 23
+; SI-NEXT: s_lshl_b32 s16, s16, 8
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_and_b32 s17, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v56, 21
+; SI-NEXT: s_lshl_b32 s18, s16, 24
+; SI-NEXT: v_readlane_b32 s16, v56, 22
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_or_b32 s16, s18, s16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_add_i32 s58, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 20
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s17, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v56, 19
+; SI-NEXT: s_lshl_b32 s16, s16, 8
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_and_b32 s17, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v56, 17
+; SI-NEXT: s_lshl_b32 s18, s16, 24
+; SI-NEXT: v_readlane_b32 s16, v56, 18
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_or_b32 s16, s18, s16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_add_i32 s59, s16, 0x3000000
+; SI-NEXT: s_add_i32 s16, s76, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 29
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s97, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s96, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s60, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 28
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 27
+; SI-NEXT: v_readlane_b32 s18, v56, 26
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 25
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s61, s16, 0x3000000
+; SI-NEXT: s_add_i32 s16, s77, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 34
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s99, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s98, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s62, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 33
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 32
+; SI-NEXT: v_readlane_b32 s18, v56, 31
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 30
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s63, s16, 0x3000000
+; SI-NEXT: s_add_i32 s16, s24, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 4
+; SI-NEXT: v_readlane_b32 s18, v56, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 2
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s72, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 1
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 0
+; SI-NEXT: v_readlane_b32 s18, v56, 36
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 35
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s73, s16, 0x3000000
+; SI-NEXT: v_readlane_b32 s16, v56, 11
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 10
+; SI-NEXT: v_readlane_b32 s18, v56, 9
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 8
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s74, s16, 0x3000000
+; SI-NEXT: s_add_i32 s16, s20, 3
+; SI-NEXT: v_readlane_b32 s17, v56, 7
+; SI-NEXT: v_readlane_b32 s18, v56, 6
+; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v56, 5
+; SI-NEXT: s_and_b32 s18, s18, 0xff
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
+; SI-NEXT: s_add_i32 s6, s6, 0x3000000
+; SI-NEXT: s_add_i32 s7, s7, 0x3000000
+; SI-NEXT: s_add_i32 s8, s8, 0x3000000
; SI-NEXT: s_add_i32 s9, s9, 0x3000000
+; SI-NEXT: s_add_i32 s10, s10, 0x3000000
+; SI-NEXT: s_add_i32 s11, s11, 0x3000000
+; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
; SI-NEXT: s_add_i32 s14, s14, 0x3000000
; SI-NEXT: s_add_i32 s15, s15, 0x3000000
-; SI-NEXT: s_add_i32 s16, s16, 0x3000000
-; SI-NEXT: s_add_i32 s17, s17, 0x3000000
-; SI-NEXT: s_add_i32 s21, s21, 0x3000000
-; SI-NEXT: s_add_i32 s24, s24, 0x3000000
-; SI-NEXT: s_add_i32 s25, s25, 0x3000000
-; SI-NEXT: s_add_i32 s26, s26, 0x3000000
-; SI-NEXT: s_add_i32 s29, s29, 0x3000000
-; SI-NEXT: s_add_i32 s40, s40, 0x3000000
-; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: s_add_i32 s19, s19, 0x3000000
-; SI-NEXT: s_add_i32 s41, s41, 0x3000000
-; SI-NEXT: s_add_i32 s42, s42, 0x3000000
-; SI-NEXT: s_add_i32 s43, s43, 0x3000000
-; SI-NEXT: s_add_i32 s44, s44, 0x3000000
-; SI-NEXT: s_add_i32 s45, s45, 0x3000000
-; SI-NEXT: s_and_b32 s82, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s45, 16
-; SI-NEXT: s_and_b32 s83, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s80, s44, 16
-; SI-NEXT: s_and_b32 s85, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s84, s43, 16
-; SI-NEXT: s_and_b32 s86, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s42, 16
-; SI-NEXT: s_and_b32 s96, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s41, 16
-; SI-NEXT: s_and_b32 s97, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s70, s19, 16
-; SI-NEXT: s_and_b32 s99, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s98, s6, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s69, s18, 16
-; SI-NEXT: s_and_b32 s18, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s42, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s20, 16
-; SI-NEXT: s_and_b32 s20, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s28, 16
-; SI-NEXT: s_and_b32 s43, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s12, 16
-; SI-NEXT: s_and_b32 s12, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s27, 16
-; SI-NEXT: s_and_b32 s44, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s40, 16
-; SI-NEXT: s_and_b32 s59, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s41, s29, 16
-; SI-NEXT: s_and_b32 s27, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s65, s26, 16
-; SI-NEXT: s_and_b32 s75, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s25, 16
-; SI-NEXT: s_and_b32 s29, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s24, 16
-; SI-NEXT: s_and_b32 s24, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s21, 16
-; SI-NEXT: s_and_b32 s21, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s55, s17, 16
-; SI-NEXT: s_and_b32 s40, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s16, 16
-; SI-NEXT: s_and_b32 s92, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s15, 16
-; SI-NEXT: s_and_b32 s93, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s91, s14, 16
-; SI-NEXT: s_and_b32 s95, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s53, s13, 16
-; SI-NEXT: s_and_b32 vcc_lo, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s11, 16
-; SI-NEXT: s_and_b32 s30, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s10, 16
-; SI-NEXT: s_and_b32 s31, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_hi, s9, 16
-; SI-NEXT: s_and_b32 s35, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s51, s8, 16
-; SI-NEXT: s_and_b32 s36, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s22, 16
-; SI-NEXT: s_and_b32 s38, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s23, 16
-; SI-NEXT: s_and_b32 s39, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s37, s5, 16
-; SI-NEXT: s_and_b32 s48, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s49, s4, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s75, s16, 0x3000000
+; SI-NEXT: s_lshr_b64 s[30:31], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[50:51], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[82:83], s[74:75], 16
+; SI-NEXT: s_lshr_b64 vcc, s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[70:71], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[84:85], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[40:41], 16
+; SI-NEXT: s_lshr_b32 s49, s75, 16
+; SI-NEXT: s_lshr_b32 s51, s73, 16
+; SI-NEXT: s_lshr_b32 s53, s63, 16
+; SI-NEXT: s_lshr_b32 s55, s61, 16
+; SI-NEXT: s_lshr_b32 s65, s59, 16
+; SI-NEXT: s_lshr_b32 s35, s57, 16
+; SI-NEXT: s_lshr_b32 s79, s47, 16
+; SI-NEXT: s_lshr_b32 s89, s45, 16
+; SI-NEXT: s_lshr_b32 s91, s43, 16
+; SI-NEXT: s_lshr_b32 s93, s41, 16
+; SI-NEXT: s_lshr_b32 s95, s15, 16
+; SI-NEXT: s_lshr_b32 s31, s13, 16
+; SI-NEXT: s_lshr_b32 s17, s11, 16
+; SI-NEXT: s_lshr_b32 s92, s9, 16
+; SI-NEXT: s_lshr_b32 s78, s7, 16
+; SI-NEXT: s_lshr_b32 s39, s5, 16
; SI-NEXT: .LBB89_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s82
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s81
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s83
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s80
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s85
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s84
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s86
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s71
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s96
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s87
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s97
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s70
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s99
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s98
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s69
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s18
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s42
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s68
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s20
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s19
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s43
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s67
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s45
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s44
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s66
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s59
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s41
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s27
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s65
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s29
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s64
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s24
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s46
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s21
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s55
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s40
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s88
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s92
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s54
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s93
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s91
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s95
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s53
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, vcc_lo
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s94
-; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s30
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s52
-; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s31
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, vcc_hi
-; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s35
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s51
-; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s36
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s34
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s38
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s50
-; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s39
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s37
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s48
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s49
+; SI-NEXT: s_and_b32 s16, s74, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s82, 16
+; SI-NEXT: s_or_b32 s16, s16, s19
+; SI-NEXT: s_and_b32 s19, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s49, 16
+; SI-NEXT: s_or_b32 s19, s19, s20
+; SI-NEXT: s_and_b32 s20, s72, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s30, 16
+; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_and_b32 s21, s73, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s51, 16
+; SI-NEXT: s_or_b32 s21, s21, s22
+; SI-NEXT: s_and_b32 s22, s62, 0xffff
+; SI-NEXT: s_lshl_b32 s23, s94, 16
+; SI-NEXT: s_or_b32 s22, s22, s23
+; SI-NEXT: s_and_b32 s23, s63, 0xffff
+; SI-NEXT: s_lshl_b32 s24, s53, 16
+; SI-NEXT: s_or_b32 s23, s23, s24
+; SI-NEXT: s_and_b32 s24, s60, 0xffff
+; SI-NEXT: s_lshl_b32 s25, vcc_lo, 16
+; SI-NEXT: s_or_b32 s24, s24, s25
+; SI-NEXT: s_and_b32 s25, s61, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s55, 16
+; SI-NEXT: s_or_b32 s25, s25, s26
+; SI-NEXT: s_and_b32 s26, s58, 0xffff
+; SI-NEXT: s_lshl_b32 s27, s70, 16
+; SI-NEXT: s_or_b32 s26, s26, s27
+; SI-NEXT: s_and_b32 s27, s59, 0xffff
+; SI-NEXT: s_lshl_b32 s28, s65, 16
+; SI-NEXT: s_or_b32 s27, s27, s28
+; SI-NEXT: s_and_b32 s28, s56, 0xffff
+; SI-NEXT: s_lshl_b32 s29, s80, 16
+; SI-NEXT: s_or_b32 s28, s28, s29
+; SI-NEXT: s_and_b32 s29, s57, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s35, 16
+; SI-NEXT: s_or_b32 s29, s29, s56
+; SI-NEXT: s_and_b32 s46, s46, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s84, 16
+; SI-NEXT: s_or_b32 s46, s46, s56
+; SI-NEXT: s_and_b32 s47, s47, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s79, 16
+; SI-NEXT: s_or_b32 s47, s47, s56
+; SI-NEXT: s_and_b32 s44, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s68, 16
+; SI-NEXT: s_or_b32 s44, s44, s56
+; SI-NEXT: s_and_b32 s45, s45, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s89, 16
+; SI-NEXT: s_or_b32 s45, s45, s56
+; SI-NEXT: s_and_b32 s42, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s34, 16
+; SI-NEXT: s_or_b32 s42, s42, s56
+; SI-NEXT: s_and_b32 s43, s43, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s91, 16
+; SI-NEXT: s_or_b32 s43, s43, s56
+; SI-NEXT: s_and_b32 s40, s40, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s36, 16
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_or_b32 s40, s40, s56
+; SI-NEXT: s_and_b32 s41, s41, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s93, 16
+; SI-NEXT: s_or_b32 s11, s11, s17
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s52, 16
+; SI-NEXT: s_or_b32 s41, s41, s56
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s38, 16
+; SI-NEXT: s_or_b32 s8, s8, s17
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s92, 16
+; SI-NEXT: s_or_b32 s14, s14, s56
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s95, 16
+; SI-NEXT: s_or_b32 s9, s9, s17
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s54, 16
+; SI-NEXT: s_or_b32 s15, s15, s56
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s48, 16
+; SI-NEXT: s_or_b32 s6, s6, s17
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s78, 16
+; SI-NEXT: s_or_b32 s12, s12, s56
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s31, 16
+; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s64, 16
+; SI-NEXT: s_or_b32 s13, s13, s56
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s50, 16
+; SI-NEXT: s_or_b32 s4, s4, s17
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s39, 16
+; SI-NEXT: s_or_b32 s10, s10, s56
+; SI-NEXT: s_or_b32 s5, s5, s17
; SI-NEXT: v_readlane_b32 s30, v45, 34
-; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s19
+; SI-NEXT: v_mov_b32_e32 v2, s20
+; SI-NEXT: v_mov_b32_e32 v3, s21
+; SI-NEXT: v_mov_b32_e32 v4, s22
+; SI-NEXT: v_mov_b32_e32 v5, s23
+; SI-NEXT: v_mov_b32_e32 v6, s24
+; SI-NEXT: v_mov_b32_e32 v7, s25
+; SI-NEXT: v_mov_b32_e32 v8, s26
+; SI-NEXT: v_mov_b32_e32 v9, s27
+; SI-NEXT: v_mov_b32_e32 v10, s28
+; SI-NEXT: v_mov_b32_e32 v11, s29
+; SI-NEXT: v_mov_b32_e32 v12, s46
+; SI-NEXT: v_mov_b32_e32 v13, s47
+; SI-NEXT: v_mov_b32_e32 v14, s44
+; SI-NEXT: v_mov_b32_e32 v15, s45
+; SI-NEXT: v_mov_b32_e32 v16, s42
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v18, s40
+; SI-NEXT: v_mov_b32_e32 v19, s41
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v21, s15
+; SI-NEXT: v_mov_b32_e32 v22, s12
+; SI-NEXT: v_mov_b32_e32 v23, s13
+; SI-NEXT: v_mov_b32_e32 v24, s10
+; SI-NEXT: v_mov_b32_e32 v25, s11
+; SI-NEXT: v_mov_b32_e32 v26, s8
+; SI-NEXT: v_mov_b32_e32 v27, s9
+; SI-NEXT: v_mov_b32_e32 v28, s6
+; SI-NEXT: v_mov_b32_e32 v29, s7
+; SI-NEXT: v_mov_b32_e32 v30, s4
+; SI-NEXT: v_mov_b32_e32 v31, s5
; SI-NEXT: v_readlane_b32 s31, v45, 35
; SI-NEXT: v_readlane_b32 s99, v45, 33
; SI-NEXT: v_readlane_b32 s98, v45, 32
@@ -151137,1993 +147372,1889 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:8
-; SI-NEXT: v_mov_b32_e32 v31, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v30
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v63
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v62
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v24
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v27
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v39
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v39
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v9
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v15
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v2
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v19
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v4
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v39
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v53
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v6
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v38
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v8
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v36
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v10
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v34
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v12
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v58
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v14
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v46
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v45
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v18
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v44
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v20
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v43
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v42
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v24
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v51
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v47
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v41
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v26
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v56
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v40
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v28
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v48
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v55
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v54
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v33
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v59
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v61
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v53
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; kill: killed $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; kill: killed $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v1
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: ; kill: killed $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; kill: killed $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v38
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; kill: killed $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v43
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v55
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v44
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v35
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v34
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v57
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v38
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v33
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; kill: killed $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v57
+; SI-NEXT: ; kill: killed $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; kill: killed $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; kill: killed $vgpr32
; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; kill: killed $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; kill: killed $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; kill: killed $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; kill: killed $vgpr2
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr8
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; kill: killed $vgpr59
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; kill: killed $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr44
; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; kill: killed $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; kill: killed $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr49
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB90_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_alignbit_b32 v38, v0, v1, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(4)
+; SI-NEXT: v_or_b32_e32 v51, v1, v2
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v39
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v35, v0, v1, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v4, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(2)
+; SI-NEXT: v_or_b32_e32 v37, v1, v2
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v38
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v32, v0, v1, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v6, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: v_or_b32_e32 v33, v1, v2
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v36
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v29, v0, v1, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v8, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v25, v0, v1, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_alignbit_b32 v40, v0, v1, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v31, v1, v2
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v34
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v21, v0, v1, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v33
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_alignbit_b32 v54, v0, v1, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v46, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v18, v0, v1, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_alignbit_b32 v52, v0, v1, 16
+; SI-NEXT: v_or_b32_e32 v62, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v10
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v41, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v60, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v12
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v35, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v58, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v32, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v56, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v16
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v63, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v44, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v18
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v59, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v42, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v20
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v57, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v21
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v55, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v4, v51, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v22
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_alignbit_b32 v15, v0, v41, 16
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v50, v0, v1, 16
+; SI-NEXT: v_or_b32_e32 v47, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v4, v51, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v49
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: v_alignbit_b32 v12, v0, v53, 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v49
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v23
+; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_alignbit_b32 v48, v0, v1, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v52, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v4, v51, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v55
-; SI-NEXT: v_alignbit_b32 v9, v0, v43, 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v53
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_alignbit_b32 v37, v0, v1, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v45, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v6, v37, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: v_alignbit_b32 v8, v0, v42, 16
-; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v25
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v50, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v6, v37, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v26
+; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_alignbit_b32 v34, v0, v1, 16
+; SI-NEXT: v_or_b32_e32 v43, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v6, v37, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: v_alignbit_b32 v5, v0, v63, 16
-; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v27
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v56
-; SI-NEXT: v_alignbit_b32 v4, v1, v22, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v47
-; SI-NEXT: v_alignbit_b32 v28, v1, v2, 16
+; SI-NEXT: v_or_b32_e32 v49, v1, v2
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v58
-; SI-NEXT: v_alignbit_b32 v3, v1, v36, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v6
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v61
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v45
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v46
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v10
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v51
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v40, v38, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v40, v38, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v8, v33, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v40, v38, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v28
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v40, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v8, v33, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v54, v35, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v29
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v5, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v8, v33, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v54, v35, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v30
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v54, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v46, v31, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v54, v35, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v48
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v3, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v46, v31, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v52, v32, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v53, v1, v2
+; SI-NEXT: v_alignbit_b32 v1, v46, v31, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v52, v32, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v41, v62, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v52, v32, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v41, v62, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v50, v29, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v41, v62, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v50, v29, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v60, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v50, v29, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v60, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v48, v25, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v60, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v48, v25, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v32, v58, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v48, v25, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v32, v58, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v37, v21, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v32, v58, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v37, v21, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v63, v56, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v37, v21, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v63, v56, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v34, v18, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v63, v56, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v34, v18, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v59, v44, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v34, v18, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v59, v44, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v0, v15, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v59, v44, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v0, v15, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v57, v42, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v0, v15, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v57, v42, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v28, v12, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v57, v42, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v28, v12, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v47, v55, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v28, v12, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_alignbit_b32 v26, v1, v2, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v60
-; SI-NEXT: v_alignbit_b32 v2, v1, v13, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v61
-; SI-NEXT: v_alignbit_b32 v23, v1, v7, 16
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v45
-; SI-NEXT: v_alignbit_b32 v20, v7, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v47, v55, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v26, v9, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v47, v55, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v26, v9, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v45, v52, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v26, v9, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v45, v52, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v23, v8, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v45, v52, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v23, v8, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v43, v50, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v23, v8, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v43, v50, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v20, v5, 24
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v43, v50, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v46
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v40, v49, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v20, v5, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v40, v49, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v20, v5, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v40, v49, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v57
-; SI-NEXT: v_alignbit_b32 v1, v1, v62, 16
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v59
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v54, v5, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v33
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v54, v5, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v1, v54, v5, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v27
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v53, v3, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v41
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v53, v3, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v1, v53, v3, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v49
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v53
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v24
-; SI-NEXT: v_lshrrev_b32_e32 v33, 24, v19
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v47
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_alignbit_b32 v17, v7, v11, 16
-; SI-NEXT: v_alignbit_b32 v6, v17, v4, 24
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v6
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v17, v4, 16
-; SI-NEXT: v_alignbit_b32 v14, v7, v39, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v17, v4, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v46
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v14, v3, 24
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v41
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v35
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v14, v3, 16
-; SI-NEXT: v_alignbit_b32 v11, v7, v30, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v32
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v14, v3, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v63
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v11, v2, 24
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v59
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v51
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v57
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v11, v2, 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v47
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v7, v7, v44, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v45
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v11, v2, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v43
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v7, v1, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v40
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v7, v1, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v54
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v7, v1, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v53
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v40
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v39, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v54
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v38, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v52
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v36, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v50
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v34, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v48
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v37
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v34
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v0
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v28
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v26
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v23
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v20
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v17
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v14
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v11
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v7
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; kill: killed $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; kill: killed $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_u32 v1, v1, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: .LBB90_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB90_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v57
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
-; SI-NEXT: v_alignbit_b32 v1, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v58
-; SI-NEXT: v_alignbit_b32 v2, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v56
-; SI-NEXT: v_alignbit_b32 v3, v3, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v22
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v48
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v51
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_alignbit_b32 v4, v4, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v44
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v5, v7, v5, 16
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v13
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v42
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v7, v7, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v30
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v48, v6, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v29
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v10
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v11, v9, v0, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v9, v12, v9, 16
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
-; SI-NEXT: v_alignbit_b32 v14, v12, v0, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v46
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v12, v15, v12, 16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v19
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v29, v6, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v27
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v49, v6, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v25
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v61
+; SI-NEXT: v_alignbit_b32 v50, v6, v4, 16
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v23
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v17, v15, v0, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v41
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v15, v18, v15, 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v22
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_alignbit_b32 v25, v28, v25, 16
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v29
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v32
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v35
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v35
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v38
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v38
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v49
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v50
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v52
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v54
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v49
-; SI-NEXT: v_add_f32_e32 v51, 0x40c00000, v50
-; SI-NEXT: v_add_f32_e32 v53, 0x40c00000, v52
-; SI-NEXT: v_add_f32_e32 v55, 0x40c00000, v54
-; SI-NEXT: v_add_f32_e32 v41, 0x40c00000, v40
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v41
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v20, v18, v0, 16
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v61
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_alignbit_b32 v29, v32, v29, 16
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_alignbit_b32 v52, v8, v6, 16
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_alignbit_b32 v18, v21, v18, 16
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v27
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v23, v21, v0, 16
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v53, v6, v4, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v30
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v21
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_alignbit_b32 v55, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v23
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v54, v6, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v28
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v19
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v21, v24, v21, 16
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
-; SI-NEXT: v_alignbit_b32 v26, v24, v0, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v24
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v6
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v27
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v22
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v19
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v10
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_alignbit_b32 v42, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v21
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v40, v6, v4, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v13
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v24
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v28, v28, v0, 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v26
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v0, v32, v0, 16
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_alignbit_b32 v44, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v43, v6, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v24
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v15
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_alignbit_b32 v56, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v45, v6, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v22
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v13
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_alignbit_b32 v58, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v15
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v47, v6, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v20
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v11
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: v_alignbit_b32 v32, v35, v32, 16
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v33
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v34, v35, v34, 16
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_alignbit_b32 v60, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v57, v6, v4, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v18
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v35
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_alignbit_b32 v62, v8, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v11
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v59, v6, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v16
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v35
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v35
-; SI-NEXT: v_alignbit_b32 v35, v38, v35, 16
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v36
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v37, v38, v37, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v36, 24, v36
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_alignbit_b32 v31, v7, v6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v63, v6, v4, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v36, 24, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 24, v30
-; SI-NEXT: v_mov_b32_e32 v27, v36
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v14
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v6
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v33, v6, v5, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v7
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v32, v5, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v12
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v38
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v39, 0x40c00000, v38
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v37, v6, v3, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v35, v3, v4, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v38
-; SI-NEXT: v_alignbit_b32 v38, v49, v38, 16
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v39, 24, v39
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v48, v49, v48, 16
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v51, v6, v1, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; SI-NEXT: v_alignbit_b32 v41, v1, v4, 16
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v12
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_alignbit_b32 v46, v34, v1, 16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v14
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v49
-; SI-NEXT: v_alignbit_b32 v50, v50, v49, 16
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v49
-; SI-NEXT: v_alignbit_b32 v52, v52, v49, 16
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_alignbit_b32 v8, v36, v1, 16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v49
-; SI-NEXT: v_alignbit_b32 v54, v54, v49, 16
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_alignbit_b32 v6, v38, v1, 16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v49
-; SI-NEXT: v_alignbit_b32 v40, v40, v49, 16
-; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v41
-; SI-NEXT: v_alignbit_b32 v6, v40, v38, 24
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_alignbit_b32 v4, v39, v1, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v14
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v12
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v55
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v10
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v3
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v5
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v40, v38, 16
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v7
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v53
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v40, v38, 8
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v11
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v51
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v13
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v54, v35, 24
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v15
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v54, v35, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v17
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v54, v35, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v19
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v52, v32, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v21
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v52, v32, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v23
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v52, v32, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v50, v29, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v4, v51, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v4, v51, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v4, v51, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v6, v37, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v6, v37, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v6, v37, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v50, v29, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v8, v33, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v50, v29, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v8, v33, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v48, v25, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v8, v33, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v48, v25, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v46, v31, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v48, v25, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v46, v31, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v37, v21, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v46, v31, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v37, v21, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v41, v62, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v37, v21, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v41, v62, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v34, v18, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v41, v62, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v34, v18, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v60, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v34, v18, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v60, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v0, v15, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v60, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v0, v15, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v32, v58, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v0, v15, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v32, v58, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v28, v12, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v32, v58, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v28, v12, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v63, v56, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v28, v12, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v63, v56, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v26, v9, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v63, v56, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v26, v9, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v59, v44, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v26, v9, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v59, v44, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v23, v8, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v59, v44, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v23, v8, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v57, v42, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v23, v8, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v57, v42, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v20, v5, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v57, v42, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v20, v5, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v47, v55, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v20, v5, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v47, v55, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v17, v4, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v47, v55, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v17, v4, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v45, v52, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v17, v4, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v45, v52, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v14, v3, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v45, v52, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v14, v3, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v43, v50, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v14, v3, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v43, v50, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v11, v2, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v43, v50, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v11, v2, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v40, v49, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v11, v2, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v40, v49, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v7, v1, 24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v40, v49, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v7, v1, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v54, v29, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v54, v29, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v1, v54, v29, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v53, v48, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v7, v1, 8
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v53, v48, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v1, v53, v48, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v40
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v54
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v6
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v52
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v50
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v46
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v48
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v41
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v37
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v35
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v34
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v32
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v0
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v63
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v28
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v59
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v26
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v57
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v23
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v47
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v20
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v45
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v17
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v43
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v14
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v40
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v11
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v54
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 8, v7
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v53
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: .LBB90_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v38
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v51
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v40
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v39
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:4
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v35
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v37
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:8
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v54
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v6
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v38
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:12
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v32
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v33
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v52
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v8
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v36
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:20
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v29
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v31
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:24
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v50
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v46
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v34
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:28
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v25
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v62
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:32
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v48
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v41
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:36
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v21
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v60
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:40
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v37
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v35
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:44
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v18
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v58
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:48
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v34
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v27
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v32
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:52
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v15
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v56
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v13, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: buffer_store_dword v6, v31, s[0:3], 0 offen offset:56
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v33
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v63
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v10, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:60
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v12
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v44
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v10, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:64
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:64
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v28
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v59
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v59
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v10, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:68
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:68
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v42
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v9, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:72
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v26
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v57
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v9, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:76
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v55
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:80
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v23
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v47
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:84
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v5
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v52
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:88
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v20
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v45
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:92
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v50
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v4, v5, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v4
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:96
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v17
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v43
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v4, v5, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v4
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:100
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v49
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:104
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v14
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v40
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:108
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:112
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v11
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v54
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v3, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:116
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:120
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v7
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v53
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: buffer_store_dword v0, v31, s[0:3], 0 offen offset:124
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
@@ -158398,1937 +154529,1854 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-LABEL: bitcast_v64bf16_to_v128i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_writelane_b32 v63, s34, 0
-; SI-NEXT: v_writelane_b32 v63, s35, 1
-; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s37, 3
-; SI-NEXT: v_writelane_b32 v63, s38, 4
-; SI-NEXT: v_writelane_b32 v63, s39, 5
-; SI-NEXT: v_writelane_b32 v63, s48, 6
-; SI-NEXT: v_writelane_b32 v63, s49, 7
-; SI-NEXT: v_writelane_b32 v63, s50, 8
-; SI-NEXT: v_writelane_b32 v63, s51, 9
-; SI-NEXT: v_writelane_b32 v63, s52, 10
-; SI-NEXT: v_writelane_b32 v63, s53, 11
-; SI-NEXT: v_writelane_b32 v63, s54, 12
-; SI-NEXT: v_writelane_b32 v63, s55, 13
-; SI-NEXT: v_writelane_b32 v63, s64, 14
-; SI-NEXT: v_writelane_b32 v63, s65, 15
-; SI-NEXT: v_writelane_b32 v63, s66, 16
-; SI-NEXT: v_writelane_b32 v63, s67, 17
-; SI-NEXT: v_writelane_b32 v63, s68, 18
-; SI-NEXT: v_writelane_b32 v63, s69, 19
-; SI-NEXT: v_writelane_b32 v63, s70, 20
-; SI-NEXT: v_writelane_b32 v63, s71, 21
-; SI-NEXT: v_writelane_b32 v63, s80, 22
-; SI-NEXT: v_writelane_b32 v63, s81, 23
-; SI-NEXT: v_writelane_b32 v63, s82, 24
-; SI-NEXT: v_writelane_b32 v63, s83, 25
-; SI-NEXT: v_writelane_b32 v63, s84, 26
-; SI-NEXT: v_writelane_b32 v63, s85, 27
-; SI-NEXT: v_writelane_b32 v63, s86, 28
-; SI-NEXT: v_writelane_b32 v63, s87, 29
-; SI-NEXT: v_writelane_b32 v63, s96, 30
-; SI-NEXT: v_writelane_b32 v63, s97, 31
-; SI-NEXT: v_writelane_b32 v63, s98, 32
-; SI-NEXT: v_writelane_b32 v63, s99, 33
-; SI-NEXT: v_writelane_b32 v63, s30, 34
-; SI-NEXT: v_writelane_b32 v63, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s92, v5
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s90, v6
-; SI-NEXT: s_lshl_b32 s92, s92, 16
-; SI-NEXT: v_readfirstlane_b32 s30, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s93
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s92
-; SI-NEXT: v_readfirstlane_b32 s42, v18
-; SI-NEXT: v_readfirstlane_b32 s44, v17
-; SI-NEXT: v_readfirstlane_b32 s46, v16
-; SI-NEXT: v_readfirstlane_b32 s56, v15
-; SI-NEXT: v_readfirstlane_b32 s58, v14
-; SI-NEXT: v_readfirstlane_b32 s60, v13
-; SI-NEXT: v_readfirstlane_b32 s62, v12
-; SI-NEXT: v_readfirstlane_b32 s72, v11
-; SI-NEXT: v_readfirstlane_b32 s74, v10
-; SI-NEXT: v_readfirstlane_b32 s76, v9
-; SI-NEXT: v_readfirstlane_b32 s78, v8
-; SI-NEXT: v_readfirstlane_b32 s88, v7
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: v_readfirstlane_b32 s94, v4
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v3
-; SI-NEXT: v_readfirstlane_b32 s34, v1
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s91
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v19
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s90
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s94
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s79
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s75
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s61
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s59
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s57
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s42
-; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT: ; implicit-def: $vgpr61 : SGPR spill to VGPR lane
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_writelane_b32 v20, s34, 0
+; SI-NEXT: v_writelane_b32 v20, s35, 1
+; SI-NEXT: v_writelane_b32 v20, s36, 2
+; SI-NEXT: v_writelane_b32 v20, s37, 3
+; SI-NEXT: v_writelane_b32 v20, s38, 4
+; SI-NEXT: v_writelane_b32 v20, s39, 5
+; SI-NEXT: v_writelane_b32 v20, s48, 6
+; SI-NEXT: v_writelane_b32 v20, s49, 7
+; SI-NEXT: v_writelane_b32 v20, s50, 8
+; SI-NEXT: v_writelane_b32 v20, s51, 9
+; SI-NEXT: v_writelane_b32 v20, s52, 10
+; SI-NEXT: v_writelane_b32 v20, s53, 11
+; SI-NEXT: v_writelane_b32 v20, s54, 12
+; SI-NEXT: v_writelane_b32 v20, s55, 13
+; SI-NEXT: v_writelane_b32 v20, s64, 14
+; SI-NEXT: v_writelane_b32 v20, s65, 15
+; SI-NEXT: v_writelane_b32 v20, s66, 16
+; SI-NEXT: v_writelane_b32 v20, s67, 17
+; SI-NEXT: v_writelane_b32 v20, s68, 18
+; SI-NEXT: v_writelane_b32 v20, s69, 19
+; SI-NEXT: v_writelane_b32 v20, s70, 20
+; SI-NEXT: v_writelane_b32 v20, s71, 21
+; SI-NEXT: v_writelane_b32 v20, s80, 22
+; SI-NEXT: v_writelane_b32 v20, s81, 23
+; SI-NEXT: v_writelane_b32 v20, s82, 24
+; SI-NEXT: v_writelane_b32 v20, s83, 25
+; SI-NEXT: v_writelane_b32 v20, s84, 26
+; SI-NEXT: v_writelane_b32 v20, s85, 27
+; SI-NEXT: v_writelane_b32 v20, s86, 28
+; SI-NEXT: v_writelane_b32 v20, s87, 29
+; SI-NEXT: v_writelane_b32 v20, s96, 30
+; SI-NEXT: v_writelane_b32 v20, s97, 31
+; SI-NEXT: v_writelane_b32 v20, s98, 32
+; SI-NEXT: v_writelane_b32 v20, s99, 33
+; SI-NEXT: v_writelane_b32 v20, s30, 34
+; SI-NEXT: v_writelane_b32 v20, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s13, v14
+; SI-NEXT: v_readfirstlane_b32 s44, v6
+; SI-NEXT: s_lshr_b32 s45, s13, 16
+; SI-NEXT: ; implicit-def: $vgpr23 : SGPR spill to VGPR lane
+; SI-NEXT: v_readfirstlane_b32 s12, v12
+; SI-NEXT: s_lshr_b32 s39, s44, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v23, s44, 0
+; SI-NEXT: v_readfirstlane_b32 s46, v3
+; SI-NEXT: s_lshr_b32 s47, s12, 16
+; SI-NEXT: v_writelane_b32 v23, s45, 1
+; SI-NEXT: v_readfirstlane_b32 s48, v10
+; SI-NEXT: s_lshr_b32 s63, s46, 16
+; SI-NEXT: v_writelane_b32 v23, s46, 2
+; SI-NEXT: s_lshr_b32 s35, s48, 16
+; SI-NEXT: v_writelane_b32 v23, s47, 3
+; SI-NEXT: v_readfirstlane_b32 s80, v8
+; SI-NEXT: v_writelane_b32 v23, s34, 4
+; SI-NEXT: s_lshr_b32 s37, s80, 16
+; SI-NEXT: v_writelane_b32 v23, s35, 5
+; SI-NEXT: v_writelane_b32 v23, s36, 6
+; SI-NEXT: v_writelane_b32 v23, s37, 7
+; SI-NEXT: v_readfirstlane_b32 s61, v11
+; SI-NEXT: v_readfirstlane_b32 s82, v4
+; SI-NEXT: v_writelane_b32 v23, s38, 8
+; SI-NEXT: s_lshr_b32 s50, s61, 16
+; SI-NEXT: s_lshr_b32 s51, s82, 16
+; SI-NEXT: v_writelane_b32 v23, s39, 9
+; SI-NEXT: v_readfirstlane_b32 s96, v2
+; SI-NEXT: v_writelane_b32 v23, s50, 10
+; SI-NEXT: s_lshr_b32 s65, s96, 16
+; SI-NEXT: v_writelane_b32 v23, s51, 11
+; SI-NEXT: v_writelane_b32 v23, s64, 12
+; SI-NEXT: s_lshr_b32 s85, s29, 16
+; SI-NEXT: v_writelane_b32 v23, s65, 13
+; SI-NEXT: v_writelane_b32 v23, s84, 14
+; SI-NEXT: s_lshr_b32 s89, s27, 16
+; SI-NEXT: v_writelane_b32 v23, s85, 15
+; SI-NEXT: v_writelane_b32 v23, s88, 16
+; SI-NEXT: s_mov_b32 s56, s17
+; SI-NEXT: s_lshr_b32 s57, s25, 16
+; SI-NEXT: v_writelane_b32 v23, s89, 17
+; SI-NEXT: s_lshr_b32 s81, s56, 16
+; SI-NEXT: v_writelane_b32 v23, s56, 18
+; SI-NEXT: v_readfirstlane_b32 s42, v1
+; SI-NEXT: s_lshr_b32 s43, s23, 16
+; SI-NEXT: v_writelane_b32 v23, s57, 19
+; SI-NEXT: s_lshr_b32 s62, s42, 16
+; SI-NEXT: v_writelane_b32 v23, s42, 20
+; SI-NEXT: s_lshr_b32 s83, s21, 16
+; SI-NEXT: v_writelane_b32 v23, s43, 21
+; SI-NEXT: v_writelane_b32 v23, s82, 22
+; SI-NEXT: s_lshr_b32 s59, s19, 16
+; SI-NEXT: v_writelane_b32 v23, s83, 23
+; SI-NEXT: v_writelane_b32 v23, s58, 24
+; SI-NEXT: v_writelane_b32 v23, s59, 25
+; SI-NEXT: s_lshr_b32 s5, s16, 16
+; SI-NEXT: v_writelane_b32 v23, s16, 26
+; SI-NEXT: v_writelane_b32 v23, s5, 28
+; SI-NEXT: v_writelane_b32 v23, s19, 30
+; SI-NEXT: s_lshr_b32 s8, s18, 16
+; SI-NEXT: v_writelane_b32 v23, s18, 32
+; SI-NEXT: s_lshr_b32 s14, s22, 16
+; SI-NEXT: v_writelane_b32 v23, s8, 34
+; SI-NEXT: s_lshr_b32 s17, s28, 16
+; SI-NEXT: v_writelane_b32 v23, s14, 36
+; SI-NEXT: v_writelane_b32 v23, s17, 38
+; SI-NEXT: v_writelane_b32 v23, s62, 40
+; SI-NEXT: v_readfirstlane_b32 s52, v18
+; SI-NEXT: v_readfirstlane_b32 s75, v17
+; SI-NEXT: v_readfirstlane_b32 s66, v16
+; SI-NEXT: v_readfirstlane_b32 s31, v15
+; SI-NEXT: v_readfirstlane_b32 s97, v13
+; SI-NEXT: v_readfirstlane_b32 s49, v9
+; SI-NEXT: v_readfirstlane_b32 s98, v7
+; SI-NEXT: v_readfirstlane_b32 s94, v5
+; SI-NEXT: v_writelane_b32 v23, s63, 41
+; SI-NEXT: s_lshr_b32 s87, s26, 16
+; SI-NEXT: s_lshr_b32 s71, s24, 16
+; SI-NEXT: s_lshr_b32 s55, s20, 16
+; SI-NEXT: s_lshr_b32 s53, s75, 16
+; SI-NEXT: s_lshr_b32 s95, s31, 16
+; SI-NEXT: s_lshr_b32 s7, s97, 16
+; SI-NEXT: s_lshr_b32 s91, s49, 16
+; SI-NEXT: s_lshr_b32 s93, s98, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s94, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v19
+; SI-NEXT: s_lshr_b32 s11, s52, 16
+; SI-NEXT: s_lshr_b32 s15, s66, 16
+; SI-NEXT: v_writelane_b32 v23, s21, 42
+; SI-NEXT: s_mov_b32 s92, s29
+; SI-NEXT: s_mov_b32 s67, s28
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: v_writelane_b32 v23, s80, 43
+; SI-NEXT: v_writelane_b32 v23, s81, 44
+; SI-NEXT: ; implicit-def: $vgpr21 : SGPR spill to VGPR lane
+; SI-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: v_mov_b32_e32 v26, v37
-; SI-NEXT: s_lshr_b64 s[98:99], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: s_lshr_b32 s87, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v60
-; SI-NEXT: v_readfirstlane_b32 s86, v25
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: v_mov_b32_e32 v25, v55
-; SI-NEXT: s_lshr_b64 s[84:85], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v25
-; SI-NEXT: s_lshr_b32 s51, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v47
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: v_mov_b32_e32 v55, v30
-; SI-NEXT: s_lshr_b64 s[70:71], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v55
-; SI-NEXT: v_mov_b32_e32 v37, v22
-; SI-NEXT: v_mov_b32_e32 v22, v33
-; SI-NEXT: v_mov_b32_e32 v33, v39
-; SI-NEXT: v_mov_b32_e32 v39, v52
-; SI-NEXT: v_mov_b32_e32 v52, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s67, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v12
-; SI-NEXT: v_mov_b32_e32 v12, v31
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s42, v3
-; SI-NEXT: v_mov_b32_e32 v3, v36
-; SI-NEXT: v_mov_b32_e32 v36, v5
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v14
-; SI-NEXT: s_lshr_b64 s[64:65], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v37
-; SI-NEXT: s_lshr_b32 s75, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v4
-; SI-NEXT: s_lshr_b64 s[48:49], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v28
-; SI-NEXT: s_lshr_b32 s43, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v48
-; SI-NEXT: v_mov_b32_e32 v29, v34
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: s_lshr_b64 s[94:95], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v22
-; SI-NEXT: v_mov_b32_e32 v30, v9
-; SI-NEXT: v_readfirstlane_b32 s74, v10
-; SI-NEXT: s_lshr_b32 s89, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v30
-; SI-NEXT: v_mov_b32_e32 v10, v51
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v10
-; SI-NEXT: s_lshr_b64 s[76:77], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v33
-; SI-NEXT: v_mov_b32_e32 v51, v1
-; SI-NEXT: s_lshr_b32 s37, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v51
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v53
-; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v39
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s15, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v49
-; SI-NEXT: v_mov_b32_e32 v1, v54
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v52
-; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s14, v36
-; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v6
-; SI-NEXT: s_lshr_b64 s[96:97], s[86:87], 16
-; SI-NEXT: s_mov_b32 s99, s96
-; SI-NEXT: v_readfirstlane_b32 s50, v56
-; SI-NEXT: s_lshr_b64 s[82:83], s[50:51], 16
-; SI-NEXT: s_mov_b32 s85, s82
-; SI-NEXT: v_readfirstlane_b32 s66, v15
-; SI-NEXT: s_lshr_b64 s[68:69], s[66:67], 16
-; SI-NEXT: s_mov_b32 s71, s68
-; SI-NEXT: s_lshr_b64 s[54:55], s[74:75], 16
-; SI-NEXT: s_mov_b32 s65, s54
-; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
-; SI-NEXT: s_mov_b32 s49, vcc_lo
-; SI-NEXT: v_readfirstlane_b32 s88, v3
-; SI-NEXT: s_lshr_b64 s[80:81], s[88:89], 16
-; SI-NEXT: s_mov_b32 s95, s80
-; SI-NEXT: v_readfirstlane_b32 s36, v13
-; SI-NEXT: s_lshr_b64 s[38:39], s[36:37], 16
-; SI-NEXT: s_mov_b32 s77, s38
-; SI-NEXT: s_mov_b32 s31, s92
-; SI-NEXT: v_readfirstlane_b32 s72, v12
-; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
-; SI-NEXT: s_mov_b32 s79, s60
-; SI-NEXT: v_readfirstlane_b32 s44, v44
-; SI-NEXT: v_readfirstlane_b32 s28, v41
-; SI-NEXT: v_readfirstlane_b32 s22, v50
-; SI-NEXT: v_readfirstlane_b32 s18, v35
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s12, v20
-; SI-NEXT: v_mov_b32_e32 v19, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s4, v31
-; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 16
-; SI-NEXT: v_lshrrev_b32_e32 v9, 24, v24
-; SI-NEXT: v_mov_b32_e32 v23, v58
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: s_lshr_b32 s59, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v46
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v45
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v24
-; SI-NEXT: s_lshr_b32 s45, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v43
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v42
-; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v27
-; SI-NEXT: s_lshr_b32 s29, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v40
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v8
-; SI-NEXT: s_lshr_b64 s[24:25], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v32
-; SI-NEXT: s_lshr_b32 s23, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v38
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v11
-; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v59
-; SI-NEXT: s_lshr_b32 s19, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s56, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s81, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s8, s6, s8
+; SI-NEXT: s_and_b32 s6, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s59, 16
+; SI-NEXT: s_or_b32 s9, s6, s9
+; SI-NEXT: s_and_b32 s6, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s55, 16
+; SI-NEXT: s_or_b32 s54, s6, s10
+; SI-NEXT: s_and_b32 s6, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s83, 16
+; SI-NEXT: s_mov_b32 s88, s55
+; SI-NEXT: s_or_b32 s55, s6, s10
+; SI-NEXT: s_and_b32 s6, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s14, 16
+; SI-NEXT: s_or_b32 s68, s6, s10
+; SI-NEXT: s_and_b32 s6, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s43, 16
+; SI-NEXT: s_or_b32 s69, s6, s10
+; SI-NEXT: s_and_b32 s6, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s71, 16
+; SI-NEXT: s_or_b32 s70, s6, s10
+; SI-NEXT: s_and_b32 s6, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s57, 16
+; SI-NEXT: s_mov_b32 s58, s71
+; SI-NEXT: s_or_b32 s71, s6, s10
+; SI-NEXT: s_and_b32 s6, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s87, 16
+; SI-NEXT: s_or_b32 s86, s6, s10
+; SI-NEXT: s_and_b32 s6, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s89, 16
+; SI-NEXT: s_mov_b32 s60, s87
+; SI-NEXT: s_or_b32 s87, s6, s10
+; SI-NEXT: s_and_b32 s6, s67, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s17, 16
+; SI-NEXT: s_or_b32 s40, s6, s10
+; SI-NEXT: s_and_b32 s6, s92, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s85, 16
+; SI-NEXT: s_or_b32 s41, s6, s10
+; SI-NEXT: s_and_b32 s6, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s62, 16
+; SI-NEXT: s_or_b32 s16, s6, s10
+; SI-NEXT: s_and_b32 s6, s96, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s65, 16
+; SI-NEXT: s_or_b32 s17, s6, s10
+; SI-NEXT: s_and_b32 s6, s46, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s63, 16
+; SI-NEXT: s_or_b32 s18, s6, s10
+; SI-NEXT: s_and_b32 s6, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s51, 16
+; SI-NEXT: s_or_b32 s19, s6, s10
+; SI-NEXT: s_and_b32 s6, s94, 0xffff
+; SI-NEXT: s_lshl_b32 s10, vcc_hi, 16
+; SI-NEXT: s_or_b32 vcc_lo, s6, s10
+; SI-NEXT: s_and_b32 s6, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s39, 16
+; SI-NEXT: s_mov_b32 s74, vcc_hi
+; SI-NEXT: s_or_b32 vcc_hi, s6, s10
+; SI-NEXT: s_and_b32 s6, s98, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s93, 16
+; SI-NEXT: s_or_b32 s62, s6, s10
+; SI-NEXT: s_and_b32 s6, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s37, 16
+; SI-NEXT: s_or_b32 s63, s6, s10
+; SI-NEXT: s_and_b32 s6, s49, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s91, 16
+; SI-NEXT: s_or_b32 s90, s6, s10
+; SI-NEXT: s_and_b32 s6, s48, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s35, 16
+; SI-NEXT: s_mov_b32 s21, s91
+; SI-NEXT: s_or_b32 s91, s6, s10
+; SI-NEXT: s_and_b32 s6, s61, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s50, 16
+; SI-NEXT: s_or_b32 s76, s6, s10
+; SI-NEXT: s_and_b32 s6, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s47, 16
+; SI-NEXT: s_or_b32 s77, s6, s10
+; SI-NEXT: s_and_b32 s6, s97, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s7, 16
+; SI-NEXT: s_or_b32 s72, s6, s10
+; SI-NEXT: s_and_b32 s6, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s45, 16
+; SI-NEXT: s_or_b32 s73, s6, s10
+; SI-NEXT: s_and_b32 s6, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s95, 16
+; SI-NEXT: s_or_b32 s28, s6, s10
+; SI-NEXT: s_and_b32 s6, s66, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s15, 16
+; SI-NEXT: s_or_b32 s29, s6, s10
+; SI-NEXT: s_and_b32 s6, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s53, 16
+; SI-NEXT: s_or_b32 s78, s6, s10
+; SI-NEXT: s_and_b32 s6, s52, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s11, 16
+; SI-NEXT: s_or_b32 s79, s6, s10
+; SI-NEXT: s_lshr_b32 s6, s5, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 44
+; SI-NEXT: s_lshr_b32 s6, s9, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 43
+; SI-NEXT: s_lshr_b32 s6, s55, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 42
+; SI-NEXT: s_lshr_b32 s6, s69, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 41
+; SI-NEXT: s_lshr_b32 s6, s71, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 40
+; SI-NEXT: s_lshr_b32 s6, s87, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 39
+; SI-NEXT: s_lshr_b32 s6, s41, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 38
+; SI-NEXT: s_lshr_b32 s6, s17, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 37
+; SI-NEXT: s_lshr_b32 s6, s19, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 36
+; SI-NEXT: s_lshr_b32 s6, vcc_hi, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 35
+; SI-NEXT: s_lshr_b32 s6, s63, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 34
+; SI-NEXT: s_lshr_b32 s6, s91, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 33
+; SI-NEXT: s_lshr_b32 s6, s77, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 32
+; SI-NEXT: s_lshr_b32 s6, s73, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 31
+; SI-NEXT: s_lshr_b32 s6, s29, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 30
+; SI-NEXT: s_lshr_b32 s6, s79, 8
+; SI-NEXT: v_writelane_b32 v22, s6, 29
+; SI-NEXT: s_bfe_u32 s6, s43, 0x80008
+; SI-NEXT: v_writelane_b32 v22, s6, 27
+; SI-NEXT: s_bfe_u32 s6, s57, 0x80008
+; SI-NEXT: s_bfe_u32 s99, s85, 0x80008
+; SI-NEXT: s_bfe_u32 s85, s65, 0x80008
+; SI-NEXT: s_bfe_u32 s65, s51, 0x80008
+; SI-NEXT: s_bfe_u32 s57, s37, 0x80008
+; SI-NEXT: s_bfe_u32 s51, s35, 0x80008
+; SI-NEXT: s_bfe_u32 s37, s45, 0x80008
+; SI-NEXT: s_mov_b32 s45, s11
+; SI-NEXT: s_bfe_u32 s35, s11, 0x80008
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 24
+; SI-NEXT: v_writelane_b32 v23, s10, 51
+; SI-NEXT: v_writelane_b32 v23, s11, 52
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v57
-; SI-NEXT: s_lshr_b32 s13, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s5, v58
-; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: s_mov_b32 s5, s7
-; SI-NEXT: s_lshr_b64 s[8:9], s[6:7], 16
-; SI-NEXT: s_mov_b32 s6, s92
-; SI-NEXT: v_writelane_b32 v61, s4, 34
-; SI-NEXT: v_writelane_b32 v62, s6, 48
-; SI-NEXT: v_writelane_b32 v61, s5, 35
-; SI-NEXT: s_mov_b32 s5, s8
-; SI-NEXT: v_writelane_b32 v62, s7, 49
-; SI-NEXT: s_mov_b32 s14, s8
-; SI-NEXT: s_lshr_b32 s6, s8, 8
-; SI-NEXT: s_lshr_b64 s[8:9], s[98:99], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 4
-; SI-NEXT: v_writelane_b32 v62, s9, 5
-; SI-NEXT: s_lshr_b64 s[8:9], s[98:99], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 2
-; SI-NEXT: v_writelane_b32 v62, s9, 3
-; SI-NEXT: s_lshr_b64 s[8:9], s[98:99], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 0
-; SI-NEXT: v_writelane_b32 v62, s9, 1
-; SI-NEXT: s_lshr_b64 s[8:9], s[84:85], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 10
-; SI-NEXT: v_writelane_b32 v62, s9, 11
-; SI-NEXT: s_lshr_b64 s[8:9], s[84:85], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 8
-; SI-NEXT: v_writelane_b32 v62, s9, 9
-; SI-NEXT: s_lshr_b64 s[8:9], s[84:85], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 6
-; SI-NEXT: v_writelane_b32 v62, s9, 7
-; SI-NEXT: s_lshr_b64 s[8:9], s[70:71], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 16
-; SI-NEXT: v_writelane_b32 v62, s9, 17
-; SI-NEXT: s_lshr_b64 s[8:9], s[70:71], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 14
-; SI-NEXT: v_writelane_b32 v62, s9, 15
-; SI-NEXT: s_lshr_b64 s[8:9], s[70:71], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 12
-; SI-NEXT: v_writelane_b32 v62, s9, 13
-; SI-NEXT: s_lshr_b64 s[8:9], s[64:65], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 22
-; SI-NEXT: v_writelane_b32 v62, s9, 23
-; SI-NEXT: s_lshr_b64 s[8:9], s[64:65], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 20
-; SI-NEXT: v_writelane_b32 v62, s9, 21
-; SI-NEXT: s_lshr_b64 s[8:9], s[64:65], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 18
-; SI-NEXT: v_writelane_b32 v62, s9, 19
-; SI-NEXT: s_lshr_b64 s[8:9], s[48:49], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 28
-; SI-NEXT: v_writelane_b32 v62, s9, 29
-; SI-NEXT: s_lshr_b64 s[8:9], s[48:49], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 26
-; SI-NEXT: v_writelane_b32 v62, s9, 27
-; SI-NEXT: s_lshr_b64 s[8:9], s[48:49], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 24
-; SI-NEXT: v_writelane_b32 v62, s9, 25
-; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 34
-; SI-NEXT: v_writelane_b32 v62, s9, 35
-; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 32
-; SI-NEXT: v_writelane_b32 v62, s9, 33
-; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 30
-; SI-NEXT: v_writelane_b32 v62, s9, 31
-; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 40
-; SI-NEXT: v_writelane_b32 v62, s9, 41
-; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 38
-; SI-NEXT: v_writelane_b32 v62, s9, 39
-; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 36
-; SI-NEXT: v_writelane_b32 v62, s9, 37
-; SI-NEXT: s_lshr_b64 s[8:9], s[30:31], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 46
-; SI-NEXT: v_writelane_b32 v62, s9, 47
-; SI-NEXT: s_lshr_b64 s[8:9], s[30:31], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 44
-; SI-NEXT: v_writelane_b32 v62, s9, 45
-; SI-NEXT: s_lshr_b64 s[8:9], s[30:31], 8
-; SI-NEXT: v_writelane_b32 v62, s8, 42
-; SI-NEXT: v_writelane_b32 v62, s9, 43
-; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 54
-; SI-NEXT: v_writelane_b32 v62, s9, 55
-; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s58, v34
-; SI-NEXT: v_writelane_b32 v62, s8, 52
-; SI-NEXT: s_lshr_b64 s[56:57], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v62, s9, 53
-; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 8
-; SI-NEXT: s_mov_b32 s63, s56
-; SI-NEXT: v_writelane_b32 v62, s8, 50
-; SI-NEXT: v_writelane_b32 v62, s9, 51
-; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 24
-; SI-NEXT: v_writelane_b32 v62, s8, 60
-; SI-NEXT: v_writelane_b32 v62, s9, 61
-; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v62, s8, 58
-; SI-NEXT: s_lshr_b64 s[52:53], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v62, s9, 59
-; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 8
-; SI-NEXT: s_mov_b32 s47, s52
-; SI-NEXT: v_writelane_b32 v62, s8, 56
-; SI-NEXT: v_writelane_b32 v62, s9, 57
-; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 24
-; SI-NEXT: v_writelane_b32 v61, s8, 2
-; SI-NEXT: v_writelane_b32 v61, s9, 3
-; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v61, s8, 0
-; SI-NEXT: s_lshr_b64 s[26:27], s[28:29], 16
-; SI-NEXT: v_writelane_b32 v61, s9, 1
-; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 8
-; SI-NEXT: s_mov_b32 s41, s26
-; SI-NEXT: v_writelane_b32 v62, s8, 62
-; SI-NEXT: v_writelane_b32 v62, s9, 63
-; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v61, s8, 8
-; SI-NEXT: v_writelane_b32 v61, s9, 9
-; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v61, s8, 6
-; SI-NEXT: s_lshr_b64 s[20:21], s[22:23], 16
-; SI-NEXT: v_writelane_b32 v61, s9, 7
-; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 8
-; SI-NEXT: s_mov_b32 s25, s20
-; SI-NEXT: v_writelane_b32 v61, s8, 4
-; SI-NEXT: v_writelane_b32 v61, s9, 5
-; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 24
-; SI-NEXT: v_writelane_b32 v61, s8, 14
-; SI-NEXT: v_writelane_b32 v61, s9, 15
-; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
-; SI-NEXT: v_writelane_b32 v61, s8, 12
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: v_writelane_b32 v61, s9, 13
-; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 8
-; SI-NEXT: s_mov_b32 s17, s34
-; SI-NEXT: v_writelane_b32 v61, s8, 10
-; SI-NEXT: v_writelane_b32 v61, s9, 11
-; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
-; SI-NEXT: v_writelane_b32 v61, s8, 20
-; SI-NEXT: v_writelane_b32 v61, s9, 21
-; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: v_writelane_b32 v61, s8, 18
-; SI-NEXT: s_lshr_b64 s[90:91], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v61, s9, 19
-; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_mov_b32 s11, s90
-; SI-NEXT: v_writelane_b32 v61, s8, 16
-; SI-NEXT: v_writelane_b32 v61, s9, 17
-; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v61, s8, 26
-; SI-NEXT: v_writelane_b32 v61, s9, 27
-; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v61, s8, 24
-; SI-NEXT: v_writelane_b32 v61, s9, 25
-; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 8
-; SI-NEXT: v_writelane_b32 v61, s8, 22
-; SI-NEXT: v_writelane_b32 v61, s9, 23
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v61, s8, 32
-; SI-NEXT: v_writelane_b32 v61, s9, 33
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v61, s8, 30
-; SI-NEXT: v_writelane_b32 v61, s9, 31
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 8
-; SI-NEXT: v_mov_b32_e32 v16, v6
-; SI-NEXT: v_writelane_b32 v61, s8, 28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v35, 24, v5
-; SI-NEXT: v_mov_b32_e32 v6, v24
-; SI-NEXT: v_mov_b32_e32 v24, v57
-; SI-NEXT: v_mov_b32_e32 v14, v21
-; SI-NEXT: v_mov_b32_e32 v15, v2
-; SI-NEXT: v_mov_b32_e32 v2, v20
-; SI-NEXT: v_mov_b32_e32 v7, v18
-; SI-NEXT: s_lshr_b32 s88, s96, 8
-; SI-NEXT: s_lshr_b32 s61, s82, 8
-; SI-NEXT: s_lshr_b32 s72, s68, 8
-; SI-NEXT: s_lshr_b32 s57, s54, 8
-; SI-NEXT: s_lshr_b32 s58, vcc_lo, 8
-; SI-NEXT: s_lshr_b32 s93, s80, 8
-; SI-NEXT: s_lshr_b32 s44, s38, 8
-; SI-NEXT: s_lshr_b32 s27, s92, 8
-; SI-NEXT: s_mov_b32 s86, s60
-; SI-NEXT: s_lshr_b32 s28, s60, 8
-; SI-NEXT: s_mov_b32 s74, s56
-; SI-NEXT: s_lshr_b32 s21, s56, 8
-; SI-NEXT: s_lshr_b32 s22, s52, 8
-; SI-NEXT: s_mov_b32 s66, s26
-; SI-NEXT: s_lshr_b32 s56, s26, 8
-; SI-NEXT: s_mov_b32 s42, s20
-; SI-NEXT: s_lshr_b32 s18, s20, 8
-; SI-NEXT: s_lshr_b32 s91, s34, 8
-; SI-NEXT: s_mov_b32 s36, s90
-; SI-NEXT: s_lshr_b32 s12, s90, 8
-; SI-NEXT: v_writelane_b32 v61, s9, 29
-; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v26
-; SI-NEXT: v_lshrrev_b32_e32 v47, 24, v25
-; SI-NEXT: v_lshrrev_b32_e32 v46, 24, v55
-; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v37
-; SI-NEXT: v_lshrrev_b32_e32 v44, 24, v28
-; SI-NEXT: v_lshrrev_b32_e32 v43, 24, v22
-; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v33
-; SI-NEXT: v_lshrrev_b32_e32 v41, 24, v39
-; SI-NEXT: v_lshrrev_b32_e32 v40, 24, v52
-; SI-NEXT: v_mov_b32_e32 v18, v27
-; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v27
-; SI-NEXT: v_mov_b32_e32 v20, v32
-; SI-NEXT: v_lshrrev_b32_e32 v54, 24, v32
-; SI-NEXT: v_mov_b32_e32 v21, v59
-; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v59
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v24
-; SI-NEXT: v_lshrrev_b32_e32 v60, 24, v23
-; SI-NEXT: v_mov_b32_e32 v27, v3
-; SI-NEXT: v_mov_b32_e32 v32, v13
-; SI-NEXT: v_mov_b32_e32 v59, v30
-; SI-NEXT: v_mov_b32_e32 v24, v10
-; SI-NEXT: v_mov_b32_e32 v38, v12
-; SI-NEXT: v_mov_b32_e32 v12, v35
-; SI-NEXT: v_mov_b32_e32 v35, v49
-; SI-NEXT: v_mov_b32_e32 v13, v28
-; SI-NEXT: v_mov_b32_e32 v28, v33
-; SI-NEXT: v_mov_b32_e32 v33, v39
-; SI-NEXT: v_mov_b32_e32 v39, v52
-; SI-NEXT: v_mov_b32_e32 v49, v5
+; SI-NEXT: v_writelane_b32 v23, s10, 49
+; SI-NEXT: v_writelane_b32 v23, s11, 50
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 8
+; SI-NEXT: v_writelane_b32 v23, s10, 47
+; SI-NEXT: v_writelane_b32 v23, s11, 48
+; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 24
+; SI-NEXT: v_writelane_b32 v23, s10, 61
+; SI-NEXT: v_writelane_b32 v23, s11, 62
+; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
+; SI-NEXT: v_writelane_b32 v23, s10, 59
+; SI-NEXT: v_writelane_b32 v23, s11, 60
+; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 8
+; SI-NEXT: v_writelane_b32 v23, s10, 57
+; SI-NEXT: v_writelane_b32 v23, s11, 58
+; SI-NEXT: s_lshr_b64 s[10:11], s[54:55], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 7
+; SI-NEXT: v_writelane_b32 v21, s11, 8
+; SI-NEXT: s_lshr_b64 s[10:11], s[54:55], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 5
+; SI-NEXT: v_writelane_b32 v21, s11, 6
+; SI-NEXT: s_lshr_b64 s[10:11], s[54:55], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 3
+; SI-NEXT: v_writelane_b32 v21, s11, 4
+; SI-NEXT: s_lshr_b64 s[10:11], s[68:69], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 17
+; SI-NEXT: v_writelane_b32 v21, s11, 18
+; SI-NEXT: s_lshr_b64 s[10:11], s[68:69], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 15
+; SI-NEXT: v_writelane_b32 v21, s11, 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[68:69], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 13
+; SI-NEXT: v_writelane_b32 v21, s11, 14
+; SI-NEXT: s_lshr_b64 s[10:11], s[70:71], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 25
+; SI-NEXT: v_writelane_b32 v21, s11, 26
+; SI-NEXT: s_lshr_b64 s[10:11], s[70:71], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 23
+; SI-NEXT: v_writelane_b32 v21, s11, 24
+; SI-NEXT: s_lshr_b64 s[10:11], s[70:71], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 21
+; SI-NEXT: v_writelane_b32 v21, s11, 22
+; SI-NEXT: s_lshr_b64 s[10:11], s[86:87], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 31
+; SI-NEXT: v_writelane_b32 v21, s11, 32
+; SI-NEXT: s_lshr_b64 s[10:11], s[86:87], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 29
+; SI-NEXT: v_writelane_b32 v21, s11, 30
+; SI-NEXT: s_lshr_b64 s[10:11], s[86:87], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 27
+; SI-NEXT: v_writelane_b32 v21, s11, 28
+; SI-NEXT: s_lshr_b64 s[10:11], s[40:41], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 37
+; SI-NEXT: v_writelane_b32 v21, s11, 38
+; SI-NEXT: s_lshr_b64 s[10:11], s[40:41], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 35
+; SI-NEXT: v_writelane_b32 v21, s11, 36
+; SI-NEXT: s_lshr_b64 s[10:11], s[40:41], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 33
+; SI-NEXT: v_writelane_b32 v21, s11, 34
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 43
+; SI-NEXT: v_writelane_b32 v21, s11, 44
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 41
+; SI-NEXT: v_writelane_b32 v21, s11, 42
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 39
+; SI-NEXT: v_writelane_b32 v21, s11, 40
+; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 49
+; SI-NEXT: v_writelane_b32 v21, s11, 50
+; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 47
+; SI-NEXT: v_writelane_b32 v21, s11, 48
+; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 45
+; SI-NEXT: v_writelane_b32 v21, s11, 46
+; SI-NEXT: s_lshr_b64 s[10:11], vcc, 24
+; SI-NEXT: v_writelane_b32 v21, s10, 55
+; SI-NEXT: v_writelane_b32 v21, s11, 56
+; SI-NEXT: s_lshr_b64 s[10:11], vcc, 16
+; SI-NEXT: v_writelane_b32 v21, s10, 53
+; SI-NEXT: v_writelane_b32 v21, s11, 54
+; SI-NEXT: s_lshr_b64 s[10:11], vcc, 8
+; SI-NEXT: v_writelane_b32 v21, s10, 51
+; SI-NEXT: v_writelane_b32 v21, s11, 52
+; SI-NEXT: s_lshr_b64 s[10:11], s[62:63], 24
+; SI-NEXT: v_writelane_b32 v21, s10, 61
+; SI-NEXT: v_writelane_b32 v21, s11, 62
+; SI-NEXT: s_lshr_b64 s[10:11], s[62:63], 16
+; SI-NEXT: v_writelane_b32 v21, s10, 59
+; SI-NEXT: v_writelane_b32 v21, s11, 60
+; SI-NEXT: s_lshr_b64 s[10:11], s[62:63], 8
+; SI-NEXT: v_writelane_b32 v21, s10, 57
+; SI-NEXT: v_writelane_b32 v22, s6, 25
+; SI-NEXT: v_writelane_b32 v21, s11, 58
+; SI-NEXT: s_lshr_b64 s[10:11], s[90:91], 24
+; SI-NEXT: v_writelane_b32 v22, s10, 3
+; SI-NEXT: s_bfe_u32 s36, s47, 0x80008
+; SI-NEXT: s_mov_b32 s47, s15
+; SI-NEXT: s_bfe_u32 s34, s15, 0x80008
+; SI-NEXT: v_writelane_b32 v22, s11, 4
+; SI-NEXT: s_lshr_b64 s[14:15], s[90:91], 16
+; SI-NEXT: v_writelane_b32 v22, s14, 1
+; SI-NEXT: v_writelane_b32 v22, s15, 2
+; SI-NEXT: s_lshr_b64 s[14:15], s[90:91], 8
+; SI-NEXT: v_writelane_b32 v21, s14, 63
+; SI-NEXT: v_writelane_b32 v22, s15, 0
+; SI-NEXT: s_lshr_b64 s[10:11], s[76:77], 24
+; SI-NEXT: v_writelane_b32 v22, s10, 9
+; SI-NEXT: v_writelane_b32 v22, s11, 10
+; SI-NEXT: s_lshr_b64 s[14:15], s[76:77], 16
+; SI-NEXT: v_writelane_b32 v22, s14, 7
+; SI-NEXT: v_writelane_b32 v22, s15, 8
+; SI-NEXT: s_lshr_b64 s[14:15], s[76:77], 8
+; SI-NEXT: v_writelane_b32 v22, s14, 5
+; SI-NEXT: v_writelane_b32 v22, s15, 6
+; SI-NEXT: s_lshr_b64 s[10:11], s[72:73], 24
+; SI-NEXT: v_writelane_b32 v23, s4, 45
+; SI-NEXT: v_writelane_b32 v22, s10, 15
+; SI-NEXT: v_writelane_b32 v23, s5, 46
+; SI-NEXT: s_mov_b32 s4, s5
+; SI-NEXT: v_writelane_b32 v22, s11, 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[72:73], 16
+; SI-NEXT: v_writelane_b32 v23, s4, 53
+; SI-NEXT: v_writelane_b32 v22, s14, 13
+; SI-NEXT: v_writelane_b32 v23, s5, 54
+; SI-NEXT: v_writelane_b32 v22, s15, 14
+; SI-NEXT: s_lshr_b64 s[14:15], s[72:73], 8
+; SI-NEXT: v_writelane_b32 v23, s8, 55
+; SI-NEXT: v_writelane_b32 v22, s14, 11
+; SI-NEXT: v_writelane_b32 v23, s9, 56
+; SI-NEXT: s_mov_b32 s4, s9
+; SI-NEXT: v_writelane_b32 v22, s15, 12
+; SI-NEXT: s_lshr_b64 s[10:11], s[28:29], 24
+; SI-NEXT: v_writelane_b32 v23, s4, 63
+; SI-NEXT: v_writelane_b32 v21, s5, 0
+; SI-NEXT: v_writelane_b32 v22, s10, 21
+; SI-NEXT: v_writelane_b32 v21, s54, 1
+; SI-NEXT: v_writelane_b32 v22, s11, 22
+; SI-NEXT: s_lshr_b64 s[14:15], s[28:29], 16
+; SI-NEXT: v_writelane_b32 v21, s55, 2
+; SI-NEXT: s_mov_b32 s4, s55
+; SI-NEXT: v_writelane_b32 v22, s14, 19
+; SI-NEXT: v_writelane_b32 v21, s4, 9
+; SI-NEXT: v_writelane_b32 v22, s15, 20
+; SI-NEXT: s_lshr_b64 s[14:15], s[28:29], 8
+; SI-NEXT: v_writelane_b32 v21, s5, 10
+; SI-NEXT: v_writelane_b32 v22, s14, 17
+; SI-NEXT: v_writelane_b32 v21, s68, 11
+; SI-NEXT: s_bfe_u32 s43, s39, 0x80008
+; SI-NEXT: v_writelane_b32 v22, s15, 18
+; SI-NEXT: s_lshr_b64 s[38:39], s[78:79], 8
+; SI-NEXT: v_writelane_b32 v21, s69, 12
+; SI-NEXT: s_mov_b32 s4, s69
+; SI-NEXT: v_writelane_b32 v22, s38, 23
+; SI-NEXT: v_writelane_b32 v21, s4, 19
+; SI-NEXT: s_bfe_u32 s81, s81, 0x80008
+; SI-NEXT: s_bfe_u32 s59, s59, 0x80008
+; SI-NEXT: s_bfe_u32 s83, s83, 0x80008
+; SI-NEXT: s_bfe_u32 s89, s89, 0x80008
+; SI-NEXT: s_lshr_b64 s[10:11], s[78:79], 24
+; SI-NEXT: s_lshr_b64 s[14:15], s[78:79], 16
+; SI-NEXT: v_writelane_b32 v22, s39, 24
+; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: s_mov_b32 s55, s88
+; SI-NEXT: v_writelane_b32 v21, s5, 20
+; SI-NEXT: s_mov_b32 s68, s71
+; SI-NEXT: s_mov_b32 s71, s58
+; SI-NEXT: s_mov_b32 s8, s87
+; SI-NEXT: s_mov_b32 s87, s60
+; SI-NEXT: s_mov_b32 s30, s40
+; SI-NEXT: s_mov_b32 s6, s41
+; SI-NEXT: s_mov_b32 s64, s16
+; SI-NEXT: s_mov_b32 s4, s17
+; SI-NEXT: s_mov_b32 s54, s19
+; SI-NEXT: s_mov_b32 s84, vcc_hi
+; SI-NEXT: s_mov_b32 vcc_hi, s74
+; SI-NEXT: s_mov_b32 s88, s63
+; SI-NEXT: s_mov_b32 s74, s91
+; SI-NEXT: s_mov_b32 s91, s21
+; SI-NEXT: s_mov_b32 s60, s77
+; SI-NEXT: s_mov_b32 s58, s73
+; SI-NEXT: s_mov_b32 s40, s29
+; SI-NEXT: s_mov_b32 s16, s79
; SI-NEXT: s_branch .LBB91_3
; SI-NEXT: .LBB91_2:
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v19, v16
-; SI-NEXT: v_writelane_b32 v62, s4, 0
-; SI-NEXT: v_writelane_b32 v62, s5, 1
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v16, v6
-; SI-NEXT: v_writelane_b32 v62, s4, 2
-; SI-NEXT: v_writelane_b32 v62, s5, 3
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v15, v2
-; SI-NEXT: v_writelane_b32 v62, s4, 4
-; SI-NEXT: v_writelane_b32 v62, s5, 5
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v14, v21
-; SI-NEXT: v_writelane_b32 v62, s4, 6
-; SI-NEXT: v_writelane_b32 v62, s5, 7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: s_mov_b32 s45, s11
+; SI-NEXT: v_writelane_b32 v23, s4, 45
+; SI-NEXT: v_writelane_b32 v23, s5, 46
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v2, v20
-; SI-NEXT: v_writelane_b32 v62, s4, 8
-; SI-NEXT: v_writelane_b32 v62, s5, 9
+; SI-NEXT: s_mov_b32 s47, s15
+; SI-NEXT: v_writelane_b32 v23, s4, 47
+; SI-NEXT: v_writelane_b32 v23, s5, 48
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v21, v59
-; SI-NEXT: v_writelane_b32 v62, s4, 10
-; SI-NEXT: v_writelane_b32 v62, s5, 11
+; SI-NEXT: s_mov_b64 s[38:39], -1
+; SI-NEXT: v_writelane_b32 v23, s4, 49
+; SI-NEXT: v_writelane_b32 v23, s5, 50
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v20, v32
-; SI-NEXT: v_writelane_b32 v62, s4, 12
-; SI-NEXT: v_writelane_b32 v62, s5, 13
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr18
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr16
+; SI-NEXT: v_writelane_b32 v23, s4, 51
+; SI-NEXT: v_writelane_b32 v23, s5, 52
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v6, v24
-; SI-NEXT: v_writelane_b32 v62, s4, 14
-; SI-NEXT: v_writelane_b32 v62, s5, 15
+; SI-NEXT: v_writelane_b32 v23, s4, 53
+; SI-NEXT: v_writelane_b32 v23, s5, 54
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v29, v34
-; SI-NEXT: v_writelane_b32 v62, s4, 16
-; SI-NEXT: v_mov_b32_e32 v32, v13
-; SI-NEXT: v_mov_b32_e32 v59, v9
-; SI-NEXT: v_mov_b32_e32 v24, v51
-; SI-NEXT: v_mov_b32_e32 v51, v1
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v38, v31
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v35, v49
-; SI-NEXT: v_mov_b32_e32 v1, v54
-; SI-NEXT: v_mov_b32_e32 v13, v28
-; SI-NEXT: v_mov_b32_e32 v28, v39
-; SI-NEXT: v_mov_b32_e32 v39, v17
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_writelane_b32 v62, s5, 17
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: v_mov_b32_e32 v7, v18
-; SI-NEXT: v_writelane_b32 v62, s4, 18
-; SI-NEXT: v_writelane_b32 v62, s5, 19
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v18, v27
-; SI-NEXT: v_writelane_b32 v62, s4, 20
-; SI-NEXT: v_writelane_b32 v62, s5, 21
+; SI-NEXT: v_writelane_b32 v23, s4, 55
+; SI-NEXT: v_writelane_b32 v23, s5, 56
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_mov_b64 s[8:9], -1
-; SI-NEXT: v_writelane_b32 v62, s4, 22
-; SI-NEXT: v_writelane_b32 v62, s5, 23
+; SI-NEXT: v_writelane_b32 v23, s4, 57
+; SI-NEXT: v_writelane_b32 v23, s5, 58
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v27, v36
-; SI-NEXT: v_writelane_b32 v62, s4, 24
-; SI-NEXT: v_writelane_b32 v62, s5, 25
+; SI-NEXT: v_writelane_b32 v23, s4, 59
+; SI-NEXT: v_writelane_b32 v23, s5, 60
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v36, v5
-; SI-NEXT: v_writelane_b32 v62, s4, 26
-; SI-NEXT: v_writelane_b32 v62, s5, 27
+; SI-NEXT: v_writelane_b32 v23, s4, 61
+; SI-NEXT: v_writelane_b32 v23, s5, 62
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v26, v37
-; SI-NEXT: v_writelane_b32 v62, s4, 28
-; SI-NEXT: v_writelane_b32 v62, s5, 29
+; SI-NEXT: v_writelane_b32 v23, s4, 63
+; SI-NEXT: v_writelane_b32 v21, s5, 0
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v25, v55
-; SI-NEXT: v_writelane_b32 v62, s4, 30
-; SI-NEXT: v_writelane_b32 v62, s5, 31
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v55, v30
-; SI-NEXT: v_writelane_b32 v62, s4, 32
-; SI-NEXT: v_writelane_b32 v62, s5, 33
+; SI-NEXT: v_writelane_b32 v21, s4, 1
+; SI-NEXT: v_writelane_b32 v21, s5, 2
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v37, v22
-; SI-NEXT: v_writelane_b32 v62, s4, 34
-; SI-NEXT: v_writelane_b32 v62, s5, 35
+; SI-NEXT: v_writelane_b32 v21, s4, 3
+; SI-NEXT: v_writelane_b32 v21, s5, 4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v22, v33
-; SI-NEXT: v_writelane_b32 v62, s4, 36
-; SI-NEXT: v_writelane_b32 v62, s5, 37
+; SI-NEXT: v_writelane_b32 v21, s4, 5
+; SI-NEXT: v_writelane_b32 v21, s5, 6
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v33, v52
-; SI-NEXT: v_writelane_b32 v62, s4, 38
-; SI-NEXT: v_writelane_b32 v62, s5, 39
+; SI-NEXT: v_writelane_b32 v21, s4, 7
+; SI-NEXT: v_writelane_b32 v21, s5, 8
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 40
-; SI-NEXT: v_writelane_b32 v62, s5, 41
+; SI-NEXT: v_writelane_b32 v21, s4, 9
+; SI-NEXT: v_writelane_b32 v21, s5, 10
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 42
-; SI-NEXT: v_writelane_b32 v62, s5, 43
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 44
-; SI-NEXT: v_writelane_b32 v62, s5, 45
+; SI-NEXT: v_writelane_b32 v21, s4, 11
+; SI-NEXT: v_writelane_b32 v21, s5, 12
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 46
-; SI-NEXT: v_writelane_b32 v62, s5, 47
+; SI-NEXT: v_writelane_b32 v21, s4, 13
+; SI-NEXT: v_writelane_b32 v21, s5, 14
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 48
-; SI-NEXT: v_writelane_b32 v62, s5, 49
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 50
-; SI-NEXT: v_writelane_b32 v62, s5, 51
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 52
-; SI-NEXT: v_writelane_b32 v62, s5, 53
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 54
-; SI-NEXT: v_writelane_b32 v62, s5, 55
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 56
-; SI-NEXT: v_writelane_b32 v62, s5, 57
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 58
-; SI-NEXT: v_writelane_b32 v62, s5, 59
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 60
-; SI-NEXT: v_writelane_b32 v62, s5, 61
+; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 62
-; SI-NEXT: v_writelane_b32 v62, s5, 63
+; SI-NEXT: v_writelane_b32 v21, s4, 15
+; SI-NEXT: v_writelane_b32 v21, s5, 16
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 0
-; SI-NEXT: v_writelane_b32 v61, s5, 1
+; SI-NEXT: v_writelane_b32 v21, s4, 17
+; SI-NEXT: v_writelane_b32 v21, s5, 18
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 2
-; SI-NEXT: v_writelane_b32 v61, s5, 3
+; SI-NEXT: v_writelane_b32 v21, s4, 19
+; SI-NEXT: v_writelane_b32 v21, s5, 20
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 4
-; SI-NEXT: v_writelane_b32 v61, s5, 5
+; SI-NEXT: v_writelane_b32 v21, s4, 21
+; SI-NEXT: v_writelane_b32 v21, s5, 22
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 6
-; SI-NEXT: v_writelane_b32 v61, s5, 7
+; SI-NEXT: v_writelane_b32 v21, s4, 23
+; SI-NEXT: v_writelane_b32 v21, s5, 24
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 8
-; SI-NEXT: v_writelane_b32 v61, s5, 9
+; SI-NEXT: v_writelane_b32 v21, s4, 25
+; SI-NEXT: v_writelane_b32 v21, s5, 26
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 10
-; SI-NEXT: v_writelane_b32 v61, s5, 11
+; SI-NEXT: v_writelane_b32 v21, s4, 27
+; SI-NEXT: v_writelane_b32 v21, s5, 28
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 12
-; SI-NEXT: v_writelane_b32 v61, s5, 13
+; SI-NEXT: v_writelane_b32 v21, s4, 29
+; SI-NEXT: v_writelane_b32 v21, s5, 30
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 14
-; SI-NEXT: v_writelane_b32 v61, s5, 15
+; SI-NEXT: v_writelane_b32 v21, s4, 31
+; SI-NEXT: v_writelane_b32 v21, s5, 32
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 16
-; SI-NEXT: v_writelane_b32 v61, s5, 17
+; SI-NEXT: v_writelane_b32 v21, s4, 33
+; SI-NEXT: v_writelane_b32 v21, s5, 34
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 18
-; SI-NEXT: v_writelane_b32 v61, s5, 19
+; SI-NEXT: v_writelane_b32 v21, s4, 35
+; SI-NEXT: v_writelane_b32 v21, s5, 36
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 20
-; SI-NEXT: v_writelane_b32 v61, s5, 21
+; SI-NEXT: v_writelane_b32 v21, s4, 37
+; SI-NEXT: v_writelane_b32 v21, s5, 38
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 22
-; SI-NEXT: v_writelane_b32 v61, s5, 23
+; SI-NEXT: v_writelane_b32 v21, s4, 39
+; SI-NEXT: v_writelane_b32 v21, s5, 40
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 24
-; SI-NEXT: v_writelane_b32 v61, s5, 25
+; SI-NEXT: v_writelane_b32 v21, s4, 41
+; SI-NEXT: v_writelane_b32 v21, s5, 42
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s4, 26
-; SI-NEXT: v_writelane_b32 v61, s5, 27
-; SI-NEXT: v_writelane_b32 v61, s14, 28
-; SI-NEXT: v_writelane_b32 v61, s15, 29
-; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v21, s4, 43
+; SI-NEXT: v_writelane_b32 v21, s5, 44
+; SI-NEXT: v_writelane_b32 v21, s10, 45
+; SI-NEXT: v_writelane_b32 v21, s11, 46
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr5
-; SI-NEXT: v_writelane_b32 v61, s14, 30
-; SI-NEXT: v_writelane_b32 v61, s15, 31
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: v_writelane_b32 v61, s14, 32
-; SI-NEXT: v_writelane_b32 v61, s15, 33
-; SI-NEXT: v_writelane_b32 v61, s4, 34
-; SI-NEXT: v_writelane_b32 v61, s5, 35
-; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v21, s10, 47
+; SI-NEXT: v_writelane_b32 v21, s11, 48
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 49
+; SI-NEXT: v_writelane_b32 v21, s11, 50
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 51
+; SI-NEXT: v_writelane_b32 v21, s11, 52
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 53
+; SI-NEXT: v_writelane_b32 v21, s11, 54
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 55
+; SI-NEXT: v_writelane_b32 v21, s11, 56
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 57
+; SI-NEXT: v_writelane_b32 v21, s11, 58
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 59
+; SI-NEXT: v_writelane_b32 v21, s11, 60
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 61
+; SI-NEXT: v_writelane_b32 v21, s11, 62
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v21, s10, 63
+; SI-NEXT: v_writelane_b32 v22, s11, 0
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 1
+; SI-NEXT: v_writelane_b32 v22, s11, 2
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 3
+; SI-NEXT: v_writelane_b32 v22, s11, 4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 5
+; SI-NEXT: v_writelane_b32 v22, s11, 6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 7
+; SI-NEXT: v_writelane_b32 v22, s11, 8
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 9
+; SI-NEXT: v_writelane_b32 v22, s11, 10
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 11
+; SI-NEXT: v_writelane_b32 v22, s11, 12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 13
+; SI-NEXT: v_writelane_b32 v22, s11, 14
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 15
+; SI-NEXT: v_writelane_b32 v22, s11, 16
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 17
+; SI-NEXT: v_writelane_b32 v22, s11, 18
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 19
+; SI-NEXT: v_writelane_b32 v22, s11, 20
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 21
+; SI-NEXT: v_writelane_b32 v22, s11, 22
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: v_writelane_b32 v22, s10, 23
+; SI-NEXT: v_writelane_b32 v22, s11, 24
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB91_3: ; %Flow
-; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_and_b64 s[38:39], s[38:39], exec
; SI-NEXT: s_cselect_b32 s5, 1, 0
; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB91_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_mov_b32_e32 v13, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v1, s4, v13
+; SI-NEXT: s_lshl_b32 s4, s53, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v13
+; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v58
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_readfirstlane_b32 s4, v1
+; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
+; SI-NEXT: s_lshl_b32 s4, s52, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v13
+; SI-NEXT: s_lshl_b32 s4, s45, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v13
+; SI-NEXT: v_readfirstlane_b32 s4, v1
+; SI-NEXT: s_lshr_b32 s9, s4, 16
+; SI-NEXT: s_mov_b32 s5, s9
+; SI-NEXT: v_readfirstlane_b32 s8, v2
+; SI-NEXT: v_writelane_b32 v22, s4, 27
+; SI-NEXT: v_writelane_b32 v22, s5, 28
+; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: v_add_f32_e32 v2, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s95, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v3
+; SI-NEXT: s_lshr_b32 s9, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s47, 16
+; SI-NEXT: v_readfirstlane_b32 s8, v2
+; SI-NEXT: v_add_f32_e32 v2, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v2
+; SI-NEXT: s_lshr_b64 s[28:29], s[8:9], 16
+; SI-NEXT: s_lshr_b32 s9, s5, 16
+; SI-NEXT: s_mov_b32 s5, s9
+; SI-NEXT: s_mov_b32 s79, s4
+; SI-NEXT: v_writelane_b32 v22, s4, 25
+; SI-NEXT: v_writelane_b32 v22, s5, 26
+; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: v_readfirstlane_b32 s8, v3
+; SI-NEXT: v_add_f32_e32 v3, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: v_add_f32_e32 v4, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v4
+; SI-NEXT: s_lshr_b32 s11, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s10, v3
+; SI-NEXT: s_lshr_b64 s[72:73], s[10:11], 16
+; SI-NEXT: v_readlane_b32 s10, v23, 0
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: v_readlane_b32 s11, v23, 1
+; SI-NEXT: v_add_f32_e32 v4, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v3
+; SI-NEXT: s_lshr_b32 s11, s5, 16
+; SI-NEXT: s_mov_b32 s5, s11
+; SI-NEXT: v_writelane_b32 v23, s4, 0
+; SI-NEXT: v_writelane_b32 v23, s5, 1
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: v_readfirstlane_b32 s10, v4
+; SI-NEXT: v_add_f32_e32 v4, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s50, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v5
+; SI-NEXT: s_lshr_b32 s15, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: v_readlane_b32 s12, v23, 2
+; SI-NEXT: v_readlane_b32 s13, v23, 3
+; SI-NEXT: v_add_f32_e32 v5, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: v_readfirstlane_b32 s14, v4
+; SI-NEXT: v_add_f32_e32 v4, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v4
+; SI-NEXT: s_lshr_b32 s13, s5, 16
+; SI-NEXT: s_mov_b32 s5, s13
+; SI-NEXT: v_writelane_b32 v23, s4, 2
+; SI-NEXT: v_writelane_b32 v23, s5, 3
+; SI-NEXT: s_lshl_b32 s5, s49, 16
+; SI-NEXT: v_readfirstlane_b32 s12, v5
+; SI-NEXT: v_add_f32_e32 v5, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s91, 16
+; SI-NEXT: v_add_f32_e32 v6, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v6
+; SI-NEXT: s_lshr_b64 s[76:77], s[14:15], 16
+; SI-NEXT: s_lshr_b32 s15, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s14, v5
+; SI-NEXT: s_lshr_b64 s[90:91], s[14:15], 16
+; SI-NEXT: v_readlane_b32 s14, v23, 4
+; SI-NEXT: s_lshl_b32 s5, s48, 16
+; SI-NEXT: v_readlane_b32 s15, v23, 5
+; SI-NEXT: v_add_f32_e32 v6, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v5
+; SI-NEXT: s_lshr_b32 s15, s5, 16
+; SI-NEXT: s_mov_b32 s5, s15
+; SI-NEXT: v_writelane_b32 v23, s4, 4
+; SI-NEXT: v_writelane_b32 v23, s5, 5
+; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: v_readfirstlane_b32 s14, v6
+; SI-NEXT: v_add_f32_e32 v6, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s93, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v13
; SI-NEXT: v_readfirstlane_b32 s5, v7
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v6
+; SI-NEXT: s_lshr_b64 s[62:63], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 6
+; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 7
+; SI-NEXT: v_add_f32_e32 v7, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v6, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v6
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: s_mov_b32 s5, s17
+; SI-NEXT: v_writelane_b32 v23, s4, 6
+; SI-NEXT: v_writelane_b32 v23, s5, 7
+; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v7
+; SI-NEXT: v_add_f32_e32 v7, s5, v13
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: v_add_f32_e32 v8, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v8
+; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v7
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 8
+; SI-NEXT: s_lshl_b32 s5, s44, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 9
+; SI-NEXT: v_add_f32_e32 v8, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v7
+; SI-NEXT: s_lshr_b32 s59, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s46, 16
+; SI-NEXT: v_readfirstlane_b32 s58, v8
+; SI-NEXT: v_add_f32_e32 v8, s5, v13
+; SI-NEXT: v_readlane_b32 s5, v23, 41
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v9
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v8
+; SI-NEXT: s_lshr_b64 s[18:19], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 10
+; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 11
+; SI-NEXT: v_add_f32_e32 v9, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v8, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v8
+; SI-NEXT: s_lshr_b32 s53, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s42, 16
+; SI-NEXT: v_readfirstlane_b32 s52, v9
+; SI-NEXT: v_add_f32_e32 v9, s5, v13
+; SI-NEXT: v_readlane_b32 s5, v23, 40
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v10, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v10
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v9
+; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 12
+; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 13
+; SI-NEXT: v_add_f32_e32 v10, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v9
+; SI-NEXT: s_lshr_b32 s81, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: v_readfirstlane_b32 s80, v10
+; SI-NEXT: v_add_f32_e32 v10, s5, v13
+; SI-NEXT: v_readlane_b32 s5, v23, 38
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v11
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v10
+; SI-NEXT: s_lshr_b64 s[50:51], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 14
+; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 15
+; SI-NEXT: v_add_f32_e32 v11, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v10, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v10
+; SI-NEXT: s_lshr_b32 s93, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s26, 16
+; SI-NEXT: v_readfirstlane_b32 s92, v11
+; SI-NEXT: v_add_f32_e32 v11, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: v_add_f32_e32 v12, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v12
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v11
+; SI-NEXT: s_lshr_b64 s[86:87], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 16
+; SI-NEXT: s_lshl_b32 s5, s27, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 17
+; SI-NEXT: v_add_f32_e32 v12, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v11
+; SI-NEXT: s_lshr_b32 s43, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s24, 16
+; SI-NEXT: v_readfirstlane_b32 s42, v12
+; SI-NEXT: v_add_f32_e32 v12, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: v_add_f32_e32 v14, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v14
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v12
+; SI-NEXT: s_lshr_b64 s[70:71], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 18
+; SI-NEXT: s_lshl_b32 s5, s25, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 19
+; SI-NEXT: v_add_f32_e32 v14, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v12, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v12
+; SI-NEXT: s_lshr_b32 s89, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s22, 16
+; SI-NEXT: v_readfirstlane_b32 s88, v14
+; SI-NEXT: v_add_f32_e32 v14, s5, v13
+; SI-NEXT: v_readlane_b32 s5, v23, 36
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v15, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v15
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v14
+; SI-NEXT: s_lshr_b64 s[24:25], s[16:17], 16
+; SI-NEXT: v_readlane_b32 s16, v23, 20
+; SI-NEXT: s_lshl_b32 s5, s23, 16
+; SI-NEXT: v_readlane_b32 s17, v23, 21
+; SI-NEXT: v_add_f32_e32 v14, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: v_add_f32_e32 v17, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v17
+; SI-NEXT: s_lshr_b32 s17, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s20, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v14
+; SI-NEXT: v_add_f32_e32 v14, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s55, 16
+; SI-NEXT: v_add_f32_e32 v15, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v15
; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: s_lshr_b64 s[90:91], s[6:7], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v2
-; SI-NEXT: s_lshr_b32 s13, s6, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s8, v15
-; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_readfirstlane_b32 s12, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_mov_b32 s5, s7
-; SI-NEXT: v_writelane_b32 v61, s4, 34
-; SI-NEXT: v_writelane_b32 v61, s5, 35
-; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
-; SI-NEXT: s_mov_b32 s11, s20
-; SI-NEXT: s_mov_b32 s5, s90
-; SI-NEXT: s_lshr_b32 s12, s20, 8
-; SI-NEXT: v_lshrrev_b32_e32 v47, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v46, 24, v9
-; SI-NEXT: v_lshrrev_b32_e32 v60, 24, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_readfirstlane_b32 s8, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: s_lshr_b32 s19, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s18, v11
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_mov_b32 s17, s34
-; SI-NEXT: s_lshr_b32 s91, s34, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_lshr_b64 s[24:25], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s22, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: s_lshr_b32 s23, s6, 16
-; SI-NEXT: s_lshr_b64 s[26:27], s[22:23], 16
-; SI-NEXT: s_mov_b32 s25, s26
-; SI-NEXT: s_lshr_b32 s18, s26, 8
-; SI-NEXT: v_lshrrev_b32_e32 v54, 24, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s8, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s6, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_lshr_b64 s[40:41], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s28, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s6, v4
-; SI-NEXT: s_lshr_b32 s29, s6, 16
-; SI-NEXT: s_lshr_b64 s[56:57], s[28:29], 16
-; SI-NEXT: s_mov_b32 s41, s56
-; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s8, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_readfirstlane_b32 s6, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v6
-; SI-NEXT: s_lshr_b32 s45, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v11
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_readfirstlane_b32 s44, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_readfirstlane_b32 s8, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v49
-; SI-NEXT: v_readfirstlane_b32 s58, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v8
-; SI-NEXT: v_readfirstlane_b32 s8, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v38
-; SI-NEXT: s_lshr_b32 s59, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v39
-; SI-NEXT: v_readfirstlane_b32 s72, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v53
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: s_lshr_b64 s[78:79], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v11
-; SI-NEXT: v_readfirstlane_b32 s8, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v36
-; SI-NEXT: s_lshr_b32 s73, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v33
-; SI-NEXT: v_readfirstlane_b32 s14, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v24
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: s_lshr_b64 s[30:31], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v12
-; SI-NEXT: v_readfirstlane_b32 s8, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v32
-; SI-NEXT: s_lshr_b32 s15, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v28
-; SI-NEXT: v_readfirstlane_b32 s36, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v29
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: s_lshr_b64 s[76:77], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v14
-; SI-NEXT: v_readfirstlane_b32 s8, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v27
-; SI-NEXT: s_lshr_b32 s37, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s88, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[6:7], 16
+; SI-NEXT: v_readlane_b32 s5, v23, 42
+; SI-NEXT: v_readlane_b32 s6, v23, 22
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_readlane_b32 s7, v23, 23
+; SI-NEXT: v_add_f32_e32 v15, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: v_add_f32_e32 v14, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v14
+; SI-NEXT: s_lshr_b32 s67, s5, 16
+; SI-NEXT: v_readlane_b32 s5, v23, 32
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s66, v15
+; SI-NEXT: v_add_f32_e32 v15, s5, v13
+; SI-NEXT: v_readlane_b32 s5, v23, 34
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v16, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v16
+; SI-NEXT: s_lshr_b32 s7, s5, 16
; SI-NEXT: v_readfirstlane_b32 s6, v15
-; SI-NEXT: s_lshr_b32 s89, s6, 16
-; SI-NEXT: s_lshr_b64 s[94:95], s[8:9], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_lshr_b64 s[80:81], s[88:89], 16
-; SI-NEXT: s_mov_b32 s95, s80
-; SI-NEXT: s_lshr_b64 s[38:39], s[36:37], 16
-; SI-NEXT: s_mov_b32 s77, s38
-; SI-NEXT: s_lshr_b64 s[96:97], s[14:15], 16
-; SI-NEXT: s_mov_b32 s31, s96
-; SI-NEXT: s_lshr_b64 s[92:93], s[72:73], 16
-; SI-NEXT: s_mov_b32 s79, s92
-; SI-NEXT: s_lshr_b64 s[60:61], s[58:59], 16
-; SI-NEXT: s_mov_b32 s63, s60
-; SI-NEXT: s_lshr_b64 s[52:53], s[44:45], 16
-; SI-NEXT: s_mov_b32 s47, s52
-; SI-NEXT: s_lshr_b32 s93, s80, 8
-; SI-NEXT: s_lshr_b32 s44, s38, 8
-; SI-NEXT: s_lshr_b32 s27, s96, 8
-; SI-NEXT: s_lshr_b32 s28, s92, 8
-; SI-NEXT: s_lshr_b32 s21, s60, 8
-; SI-NEXT: s_lshr_b32 s22, s52, 8
-; SI-NEXT: s_mov_b32 s36, s20
-; SI-NEXT: s_mov_b32 s14, s90
-; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v1
-; SI-NEXT: v_lshrrev_b32_e32 v43, 24, v15
-; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v14
-; SI-NEXT: v_lshrrev_b32_e32 v41, 24, v12
-; SI-NEXT: v_lshrrev_b32_e32 v40, 24, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 24, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s8, v18
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_lshr_b64 s[48:49], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s42, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: s_lshr_b64 s[30:31], s[6:7], 16
+; SI-NEXT: v_readlane_b32 s5, v23, 30
+; SI-NEXT: v_readlane_b32 s6, v23, 24
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_readlane_b32 s7, v23, 25
+; SI-NEXT: v_add_f32_e32 v16, s5, v13
+; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: v_add_f32_e32 v15, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v15
+; SI-NEXT: s_lshr_b32 s75, s5, 16
+; SI-NEXT: v_readlane_b32 s5, v23, 26
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s74, v16
+; SI-NEXT: v_add_f32_e32 v16, s5, v13
+; SI-NEXT: v_readlane_b32 s5, v23, 28
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v18, s5, v13
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: s_lshr_b32 s7, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[74:75], 16
; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: s_lshr_b32 s43, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s8, v18
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
-; SI-NEXT: s_mov_b32 s49, vcc_lo
-; SI-NEXT: s_lshr_b32 s58, vcc_lo, 8
-; SI-NEXT: s_mov_b32 s42, s26
-; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v13
-; SI-NEXT: v_lshrrev_b32_e32 v44, 24, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v13
-; SI-NEXT: s_lshr_b32 s75, s6, 16
-; SI-NEXT: s_lshr_b64 s[64:65], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s74, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[54:55], s[74:75], 16
-; SI-NEXT: s_mov_b32 s65, s54
-; SI-NEXT: s_lshr_b32 s57, s54, 8
-; SI-NEXT: s_mov_b32 s74, s60
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s8, v18
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v9
-; SI-NEXT: s_lshr_b32 s67, s6, 16
-; SI-NEXT: s_lshr_b64 s[70:71], s[8:9], 16
-; SI-NEXT: v_lshrrev_b32_e32 v9, 24, v6
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s66, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[68:69], s[66:67], 16
+; SI-NEXT: s_mov_b32 s74, s4
+; SI-NEXT: v_add_f32_e32 v16, s5, v13
+; SI-NEXT: v_readlane_b32 s4, v23, 43
+; SI-NEXT: v_readlane_b32 s5, v23, 44
+; SI-NEXT: s_lshl_b32 s4, s5, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v13
+; SI-NEXT: s_mov_b32 s31, s48
+; SI-NEXT: v_readfirstlane_b32 s4, v13
+; SI-NEXT: s_lshr_b64 s[64:65], s[66:67], 16
+; SI-NEXT: s_lshr_b32 s57, s4, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[30:31], 24
+; SI-NEXT: s_mov_b32 s55, s64
+; SI-NEXT: v_writelane_b32 v23, s4, 61
+; SI-NEXT: v_writelane_b32 v23, s5, 62
+; SI-NEXT: s_lshr_b64 s[4:5], s[54:55], 24
+; SI-NEXT: v_writelane_b32 v21, s4, 7
+; SI-NEXT: s_lshr_b64 s[22:23], s[16:17], 16
+; SI-NEXT: v_writelane_b32 v21, s5, 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[54:55], 16
+; SI-NEXT: s_mov_b32 s25, s22
+; SI-NEXT: v_writelane_b32 v21, s4, 5
+; SI-NEXT: s_mov_b32 s63, s40
+; SI-NEXT: s_lshr_b64 s[26:27], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[88:89], 16
+; SI-NEXT: s_mov_b32 s88, s40
+; SI-NEXT: v_writelane_b32 v21, s5, 6
+; SI-NEXT: s_lshr_b64 s[40:41], s[24:25], 8
+; SI-NEXT: s_mov_b32 s87, s26
+; SI-NEXT: v_writelane_b32 v21, s40, 13
+; SI-NEXT: v_writelane_b32 v21, s41, 14
+; SI-NEXT: s_lshr_b64 s[40:41], s[86:87], 24
+; SI-NEXT: v_writelane_b32 v21, s40, 31
+; SI-NEXT: v_writelane_b32 v21, s41, 32
+; SI-NEXT: s_lshr_b64 s[40:41], s[86:87], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[92:93], 16
+; SI-NEXT: v_writelane_b32 v21, s40, 29
+; SI-NEXT: s_lshr_b64 s[38:39], s[80:81], 16
+; SI-NEXT: s_mov_b32 s51, s60
+; SI-NEXT: v_writelane_b32 v21, s41, 30
+; SI-NEXT: s_lshr_b64 s[40:41], s[86:87], 8
+; SI-NEXT: s_mov_b32 s37, s38
+; SI-NEXT: s_mov_b32 s92, s38
+; SI-NEXT: s_mov_b64 s[38:39], s[50:51]
+; SI-NEXT: v_writelane_b32 v21, s40, 27
+; SI-NEXT: v_writelane_b32 v21, s41, 28
+; SI-NEXT: s_lshr_b64 s[40:41], s[38:39], 24
+; SI-NEXT: v_writelane_b32 v21, s40, 37
+; SI-NEXT: v_writelane_b32 v21, s41, 38
+; SI-NEXT: s_lshr_b64 s[40:41], s[38:39], 16
+; SI-NEXT: v_writelane_b32 v21, s40, 35
+; SI-NEXT: v_writelane_b32 v21, s41, 36
+; SI-NEXT: s_lshr_b64 s[40:41], s[38:39], 8
+; SI-NEXT: s_mov_b64 s[20:21], s[36:37]
+; SI-NEXT: v_writelane_b32 v21, s40, 33
+; SI-NEXT: v_writelane_b32 v21, s41, 34
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
+; SI-NEXT: v_writelane_b32 v21, s40, 43
+; SI-NEXT: v_writelane_b32 v21, s41, 44
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: v_writelane_b32 v21, s40, 41
+; SI-NEXT: s_lshr_b64 s[84:85], s[52:53], 16
+; SI-NEXT: v_writelane_b32 v21, s41, 42
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 8
+; SI-NEXT: s_mov_b32 s19, s84
+; SI-NEXT: v_writelane_b32 v21, s40, 39
+; SI-NEXT: v_writelane_b32 v21, s41, 40
+; SI-NEXT: s_lshr_b64 s[40:41], s[18:19], 24
+; SI-NEXT: v_writelane_b32 v21, s40, 49
+; SI-NEXT: v_writelane_b32 v21, s41, 50
+; SI-NEXT: s_lshr_b64 s[40:41], s[18:19], 16
+; SI-NEXT: v_writelane_b32 v21, s40, 47
+; SI-NEXT: s_lshr_b64 s[44:45], s[58:59], 16
+; SI-NEXT: v_writelane_b32 v21, s41, 48
+; SI-NEXT: s_lshr_b64 s[40:41], s[18:19], 8
+; SI-NEXT: s_mov_b32 vcc_hi, s44
+; SI-NEXT: v_writelane_b32 v21, s40, 45
+; SI-NEXT: v_writelane_b32 v21, s41, 46
+; SI-NEXT: s_lshr_b64 s[40:41], vcc, 24
+; SI-NEXT: v_writelane_b32 v21, s40, 55
+; SI-NEXT: v_writelane_b32 v21, s41, 56
+; SI-NEXT: s_lshr_b64 s[40:41], vcc, 16
+; SI-NEXT: v_writelane_b32 v21, s40, 53
+; SI-NEXT: v_writelane_b32 v21, s41, 54
+; SI-NEXT: s_lshr_b64 s[40:41], vcc, 8
+; SI-NEXT: v_writelane_b32 v21, s40, 51
+; SI-NEXT: v_writelane_b32 v21, s41, 52
+; SI-NEXT: s_lshr_b64 s[40:41], s[62:63], 24
+; SI-NEXT: v_writelane_b32 v21, s40, 61
+; SI-NEXT: v_writelane_b32 v21, s41, 62
+; SI-NEXT: s_lshr_b64 s[40:41], s[62:63], 16
+; SI-NEXT: v_writelane_b32 v21, s40, 59
+; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], 16
+; SI-NEXT: v_writelane_b32 v21, s41, 60
+; SI-NEXT: s_lshr_b64 s[40:41], s[62:63], 8
+; SI-NEXT: s_mov_b32 s91, s14
+; SI-NEXT: v_writelane_b32 v21, s40, 57
+; SI-NEXT: v_writelane_b32 v21, s41, 58
+; SI-NEXT: s_lshr_b64 s[40:41], s[90:91], 24
+; SI-NEXT: v_writelane_b32 v22, s40, 3
+; SI-NEXT: v_writelane_b32 v22, s41, 4
+; SI-NEXT: s_lshr_b64 s[40:41], s[90:91], 16
+; SI-NEXT: v_writelane_b32 v22, s40, 1
+; SI-NEXT: s_lshr_b64 s[12:13], s[12:13], 16
+; SI-NEXT: v_writelane_b32 v22, s41, 2
+; SI-NEXT: s_lshr_b64 s[40:41], s[90:91], 8
+; SI-NEXT: s_mov_b32 s77, s12
+; SI-NEXT: v_writelane_b32 v21, s40, 63
+; SI-NEXT: v_writelane_b32 v22, s41, 0
+; SI-NEXT: s_lshr_b64 s[40:41], s[76:77], 24
+; SI-NEXT: v_writelane_b32 v22, s40, 9
+; SI-NEXT: v_writelane_b32 v22, s41, 10
+; SI-NEXT: s_lshr_b64 s[40:41], s[76:77], 16
+; SI-NEXT: v_writelane_b32 v22, s40, 7
+; SI-NEXT: s_lshr_b64 s[10:11], s[10:11], 16
+; SI-NEXT: v_writelane_b32 v22, s41, 8
+; SI-NEXT: s_lshr_b64 s[40:41], s[76:77], 8
+; SI-NEXT: s_mov_b32 s73, s10
+; SI-NEXT: v_writelane_b32 v22, s40, 5
+; SI-NEXT: v_writelane_b32 v22, s41, 6
+; SI-NEXT: s_lshr_b64 s[40:41], s[72:73], 24
+; SI-NEXT: v_writelane_b32 v22, s40, 15
+; SI-NEXT: v_writelane_b32 v22, s41, 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[72:73], 16
+; SI-NEXT: v_writelane_b32 v22, s40, 13
+; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
+; SI-NEXT: v_writelane_b32 v22, s41, 14
+; SI-NEXT: s_lshr_b64 s[40:41], s[72:73], 8
+; SI-NEXT: s_mov_b32 s29, s8
+; SI-NEXT: v_writelane_b32 v22, s40, 11
+; SI-NEXT: v_writelane_b32 v22, s41, 12
+; SI-NEXT: s_lshr_b64 s[40:41], s[28:29], 24
+; SI-NEXT: v_writelane_b32 v22, s40, 21
+; SI-NEXT: v_writelane_b32 v22, s41, 22
+; SI-NEXT: s_lshr_b64 s[40:41], s[28:29], 16
+; SI-NEXT: v_writelane_b32 v22, s40, 19
+; SI-NEXT: v_writelane_b32 v22, s41, 20
+; SI-NEXT: s_lshr_b64 s[40:41], s[28:29], 8
+; SI-NEXT: v_writelane_b32 v22, s40, 17
+; SI-NEXT: v_writelane_b32 v22, s41, 18
+; SI-NEXT: s_lshr_b64 s[40:41], s[78:79], 24
+; SI-NEXT: v_writelane_b32 v23, s40, 32
+; SI-NEXT: v_writelane_b32 v23, s41, 33
+; SI-NEXT: s_lshr_b64 s[40:41], s[78:79], 16
+; SI-NEXT: v_writelane_b32 v23, s40, 34
+; SI-NEXT: v_writelane_b32 v23, s41, 35
+; SI-NEXT: s_mov_b32 s16, s26
+; SI-NEXT: v_writelane_b32 v23, s16, 36
+; SI-NEXT: v_writelane_b32 v23, s17, 37
+; SI-NEXT: s_mov_b32 s16, s60
+; SI-NEXT: v_writelane_b32 v23, s16, 26
+; SI-NEXT: v_writelane_b32 v23, s17, 27
+; SI-NEXT: s_mov_b32 s16, s84
+; SI-NEXT: v_writelane_b32 v23, s16, 38
+; SI-NEXT: v_readfirstlane_b32 s56, v16
+; SI-NEXT: v_writelane_b32 v23, s17, 39
+; SI-NEXT: s_mov_b32 s16, s14
+; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
+; SI-NEXT: s_mov_b32 s66, s12
+; SI-NEXT: s_lshr_b64 s[96:97], s[56:57], 16
+; SI-NEXT: v_writelane_b32 v23, s16, 30
; SI-NEXT: s_mov_b32 s71, s68
-; SI-NEXT: s_lshr_b32 s72, s68, 8
-; SI-NEXT: s_mov_b32 s66, s56
-; SI-NEXT: s_lshr_b32 s56, s56, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s8, v18
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: s_lshr_b32 s51, s6, 16
-; SI-NEXT: s_lshr_b64 s[84:85], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s50, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[82:83], s[50:51], 16
-; SI-NEXT: s_mov_b32 s85, s82
-; SI-NEXT: s_lshr_b32 s61, s82, 8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s8, v18
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v1
-; SI-NEXT: s_lshr_b32 s87, s6, 16
-; SI-NEXT: s_lshr_b64 s[98:99], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_readfirstlane_b32 s86, v17
-; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 16
-; SI-NEXT: s_mov_b32 s99, s8
-; SI-NEXT: s_lshr_b64 s[6:7], s[98:99], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 4
-; SI-NEXT: v_writelane_b32 v62, s7, 5
-; SI-NEXT: s_lshr_b64 s[6:7], s[98:99], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 2
-; SI-NEXT: v_writelane_b32 v62, s7, 3
-; SI-NEXT: s_lshr_b64 s[6:7], s[98:99], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 0
-; SI-NEXT: v_writelane_b32 v62, s7, 1
-; SI-NEXT: s_lshr_b64 s[6:7], s[84:85], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 10
-; SI-NEXT: v_writelane_b32 v62, s7, 11
-; SI-NEXT: s_lshr_b64 s[6:7], s[84:85], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 8
-; SI-NEXT: v_writelane_b32 v62, s7, 9
-; SI-NEXT: s_lshr_b64 s[6:7], s[84:85], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 6
-; SI-NEXT: v_writelane_b32 v62, s7, 7
-; SI-NEXT: s_lshr_b64 s[6:7], s[70:71], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 16
-; SI-NEXT: v_writelane_b32 v62, s7, 17
-; SI-NEXT: s_lshr_b64 s[6:7], s[70:71], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 14
-; SI-NEXT: v_writelane_b32 v62, s7, 15
-; SI-NEXT: s_lshr_b64 s[6:7], s[70:71], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 12
-; SI-NEXT: v_writelane_b32 v62, s7, 13
-; SI-NEXT: s_lshr_b64 s[6:7], s[64:65], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 22
-; SI-NEXT: v_writelane_b32 v62, s7, 23
-; SI-NEXT: s_lshr_b64 s[6:7], s[64:65], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 20
-; SI-NEXT: v_writelane_b32 v62, s7, 21
-; SI-NEXT: s_lshr_b64 s[6:7], s[64:65], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 18
-; SI-NEXT: v_writelane_b32 v62, s7, 19
-; SI-NEXT: s_lshr_b64 s[6:7], s[48:49], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 28
-; SI-NEXT: v_writelane_b32 v62, s7, 29
-; SI-NEXT: s_lshr_b64 s[6:7], s[48:49], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 26
-; SI-NEXT: v_writelane_b32 v62, s7, 27
-; SI-NEXT: s_lshr_b64 s[6:7], s[48:49], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 24
-; SI-NEXT: v_writelane_b32 v62, s7, 25
-; SI-NEXT: s_lshr_b64 s[6:7], s[94:95], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 34
-; SI-NEXT: v_writelane_b32 v62, s7, 35
-; SI-NEXT: s_lshr_b64 s[6:7], s[94:95], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 32
-; SI-NEXT: v_writelane_b32 v62, s7, 33
-; SI-NEXT: s_lshr_b64 s[6:7], s[94:95], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 30
-; SI-NEXT: v_writelane_b32 v62, s7, 31
-; SI-NEXT: s_lshr_b64 s[6:7], s[76:77], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 40
-; SI-NEXT: v_writelane_b32 v62, s7, 41
-; SI-NEXT: s_lshr_b64 s[6:7], s[76:77], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 38
-; SI-NEXT: v_writelane_b32 v62, s7, 39
-; SI-NEXT: s_lshr_b64 s[6:7], s[76:77], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 36
-; SI-NEXT: v_writelane_b32 v62, s7, 37
-; SI-NEXT: s_lshr_b64 s[6:7], s[30:31], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 46
-; SI-NEXT: v_writelane_b32 v62, s7, 47
-; SI-NEXT: s_lshr_b64 s[6:7], s[30:31], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 44
-; SI-NEXT: v_writelane_b32 v62, s7, 45
-; SI-NEXT: s_lshr_b64 s[6:7], s[30:31], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 42
-; SI-NEXT: v_writelane_b32 v62, s7, 43
-; SI-NEXT: s_lshr_b64 s[6:7], s[78:79], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 54
-; SI-NEXT: v_writelane_b32 v62, s7, 55
-; SI-NEXT: s_lshr_b64 s[6:7], s[78:79], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 52
-; SI-NEXT: v_writelane_b32 v62, s7, 53
-; SI-NEXT: s_lshr_b64 s[6:7], s[78:79], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 50
-; SI-NEXT: v_writelane_b32 v62, s7, 51
-; SI-NEXT: s_lshr_b64 s[6:7], s[62:63], 24
-; SI-NEXT: v_writelane_b32 v62, s6, 60
-; SI-NEXT: v_writelane_b32 v62, s7, 61
-; SI-NEXT: s_lshr_b64 s[6:7], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v62, s6, 58
-; SI-NEXT: v_writelane_b32 v62, s7, 59
-; SI-NEXT: s_lshr_b64 s[6:7], s[62:63], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 56
-; SI-NEXT: v_writelane_b32 v62, s7, 57
-; SI-NEXT: s_lshr_b64 s[6:7], s[46:47], 24
-; SI-NEXT: v_writelane_b32 v61, s6, 2
-; SI-NEXT: v_writelane_b32 v61, s7, 3
-; SI-NEXT: s_lshr_b64 s[6:7], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v61, s6, 0
-; SI-NEXT: v_writelane_b32 v61, s7, 1
-; SI-NEXT: s_lshr_b64 s[6:7], s[46:47], 8
-; SI-NEXT: v_writelane_b32 v62, s6, 62
-; SI-NEXT: v_writelane_b32 v62, s7, 63
-; SI-NEXT: s_lshr_b64 s[6:7], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v61, s6, 8
-; SI-NEXT: v_writelane_b32 v61, s7, 9
-; SI-NEXT: s_lshr_b64 s[6:7], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v61, s6, 6
-; SI-NEXT: v_writelane_b32 v61, s7, 7
-; SI-NEXT: s_lshr_b64 s[6:7], s[40:41], 8
-; SI-NEXT: v_writelane_b32 v61, s6, 4
-; SI-NEXT: v_writelane_b32 v61, s7, 5
-; SI-NEXT: s_lshr_b64 s[6:7], s[24:25], 24
-; SI-NEXT: v_writelane_b32 v61, s6, 14
-; SI-NEXT: v_writelane_b32 v61, s7, 15
-; SI-NEXT: s_lshr_b64 s[6:7], s[24:25], 16
-; SI-NEXT: v_writelane_b32 v61, s6, 12
-; SI-NEXT: v_writelane_b32 v61, s7, 13
-; SI-NEXT: s_lshr_b64 s[6:7], s[24:25], 8
-; SI-NEXT: v_writelane_b32 v61, s6, 10
-; SI-NEXT: v_writelane_b32 v61, s7, 11
-; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; SI-NEXT: v_writelane_b32 v61, s6, 20
-; SI-NEXT: v_writelane_b32 v61, s7, 21
-; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: v_writelane_b32 v61, s6, 18
-; SI-NEXT: v_writelane_b32 v61, s7, 19
-; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 8
-; SI-NEXT: v_writelane_b32 v61, s6, 16
-; SI-NEXT: v_writelane_b32 v61, s7, 17
-; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v61, s6, 26
-; SI-NEXT: v_writelane_b32 v61, s7, 27
-; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v61, s6, 24
-; SI-NEXT: v_writelane_b32 v61, s7, 25
-; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 8
-; SI-NEXT: v_writelane_b32 v61, s6, 22
-; SI-NEXT: v_writelane_b32 v61, s7, 23
-; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v61, s6, 32
-; SI-NEXT: v_writelane_b32 v61, s7, 33
-; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v61, s6, 30
-; SI-NEXT: v_writelane_b32 v61, s7, 31
-; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 8
-; SI-NEXT: v_writelane_b32 v61, s6, 28
-; SI-NEXT: v_writelane_b32 v61, s7, 29
-; SI-NEXT: s_mov_b32 s6, s96
-; SI-NEXT: v_writelane_b32 v62, s6, 48
-; SI-NEXT: s_lshr_b32 s88, s8, 8
-; SI-NEXT: v_writelane_b32 v62, s7, 49
-; SI-NEXT: s_mov_b32 s96, s8
-; SI-NEXT: s_mov_b32 s86, s92
-; SI-NEXT: s_lshr_b32 s6, s90, 8
-; SI-NEXT: .LBB91_5: ; %end
-; SI-NEXT: v_readlane_b32 s8, v62, 0
-; SI-NEXT: s_and_b32 s5, s98, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 1
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 2
-; SI-NEXT: v_readlane_b32 s9, v62, 3
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s98, v62, 4
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s98, 24
+; SI-NEXT: s_mov_b32 s7, s96
+; SI-NEXT: s_lshr_b64 s[40:41], s[78:79], 8
+; SI-NEXT: v_writelane_b32 v23, s17, 31
+; SI-NEXT: s_mov_b32 s94, s66
+; SI-NEXT: s_mov_b32 s80, s44
+; SI-NEXT: s_mov_b32 s52, s10
+; SI-NEXT: s_mov_b32 s58, s8
+; SI-NEXT: s_lshr_b64 s[12:13], s[6:7], 24
+; SI-NEXT: s_lshr_b64 s[8:9], s[30:31], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[30:31], 8
+; SI-NEXT: s_lshr_b64 s[44:45], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[70:71], 24
+; SI-NEXT: v_writelane_b32 v22, s40, 23
+; SI-NEXT: v_writelane_b32 v23, s94, 28
+; SI-NEXT: s_lshr_b64 s[98:99], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[6:7], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[54:55], 8
+; SI-NEXT: s_lshr_b64 s[50:51], s[24:25], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[70:71], 16
+; SI-NEXT: s_lshr_b64 s[82:83], s[70:71], 8
+; SI-NEXT: v_writelane_b32 v22, s41, 24
+; SI-NEXT: s_lshr_b32 s56, s96, 8
+; SI-NEXT: s_lshr_b32 s47, s48, 8
+; SI-NEXT: s_lshr_b32 s45, s64, 8
+; SI-NEXT: s_lshr_b32 s42, s22, 8
+; SI-NEXT: s_lshr_b32 s41, s68, 8
+; SI-NEXT: s_lshr_b32 s40, s26, 8
+; SI-NEXT: s_lshr_b32 s27, s60, 8
+; SI-NEXT: s_mov_b32 s60, s52
+; SI-NEXT: s_lshr_b32 s25, s92, 8
+; SI-NEXT: s_lshr_b32 s23, s84, 8
+; SI-NEXT: s_mov_b32 s26, s80
+; SI-NEXT: s_lshr_b32 s21, s80, 8
+; SI-NEXT: s_lshr_b32 s19, s88, 8
+; SI-NEXT: s_lshr_b32 s16, s14, 8
+; SI-NEXT: v_writelane_b32 v23, s95, 29
+; SI-NEXT: s_lshr_b32 s15, s66, 8
+; SI-NEXT: s_lshr_b32 s13, s52, 8
+; SI-NEXT: s_lshr_b32 s11, s58, 8
+; SI-NEXT: s_mov_b32 s94, s74
+; SI-NEXT: s_lshr_b32 s9, s74, 8
+; SI-NEXT: v_lshrrev_b32_e32 v16, 24, v13
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v15
+; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v14
+; SI-NEXT: v_lshrrev_b32_e32 v13, 24, v17
+; SI-NEXT: v_lshrrev_b32_e32 v12, 24, v12
+; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v11
+; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v10
+; SI-NEXT: v_lshrrev_b32_e32 v9, 24, v9
+; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v8
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_lshrrev_b32_e32 v5, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; SI-NEXT: s_branch .LBB91_6
+; SI-NEXT: .LBB91_5:
+; SI-NEXT: v_writelane_b32 v23, s60, 28
+; SI-NEXT: v_writelane_b32 v23, s61, 29
+; SI-NEXT: v_writelane_b32 v23, s74, 30
+; SI-NEXT: v_writelane_b32 v23, s75, 31
+; SI-NEXT: v_writelane_b32 v23, s54, 38
+; SI-NEXT: v_writelane_b32 v23, s55, 39
+; SI-NEXT: s_mov_b32 s38, s30
+; SI-NEXT: v_readlane_b32 s30, v23, 55
+; SI-NEXT: v_readlane_b32 s31, v23, 56
+; SI-NEXT: v_writelane_b32 v23, s6, 26
+; SI-NEXT: v_writelane_b32 v23, s7, 27
+; SI-NEXT: v_readlane_b32 s6, v23, 45
+; SI-NEXT: v_readlane_b32 s7, v23, 46
+; SI-NEXT: v_writelane_b32 v23, s10, 32
+; SI-NEXT: v_writelane_b32 v23, s11, 33
+; SI-NEXT: v_writelane_b32 v23, s14, 34
+; SI-NEXT: v_writelane_b32 v23, s15, 35
+; SI-NEXT: v_writelane_b32 v23, s8, 36
+; SI-NEXT: v_writelane_b32 v23, s9, 37
+; SI-NEXT: s_mov_b32 s60, s58
+; SI-NEXT: s_mov_b32 s58, s40
+; SI-NEXT: v_readlane_b32 s52, v21, 15
+; SI-NEXT: v_readlane_b32 s40, v23, 8
+; SI-NEXT: v_readlane_b32 s5, v22, 25
+; SI-NEXT: s_mov_b32 s44, s52
+; SI-NEXT: v_readlane_b32 s41, v23, 9
+; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_readlane_b32 s5, v22, 27
+; SI-NEXT: v_mov_b32_e32 v15, s59
+; SI-NEXT: s_mov_b32 s94, s16
+; SI-NEXT: v_readlane_b32 s16, v21, 25
+; SI-NEXT: v_writelane_b32 v22, s44, 27
+; SI-NEXT: s_mov_b32 s59, s41
+; SI-NEXT: v_readlane_b32 s40, v23, 14
+; SI-NEXT: s_mov_b32 s46, s16
+; SI-NEXT: v_writelane_b32 v22, s45, 28
+; SI-NEXT: v_readlane_b32 s41, v23, 15
+; SI-NEXT: v_mov_b32_e32 v9, s85
+; SI-NEXT: s_mov_b32 s26, s84
+; SI-NEXT: v_readlane_b32 s24, v21, 11
+; SI-NEXT: v_readlane_b32 s22, v21, 19
+; SI-NEXT: v_readlane_b32 s8, v23, 59
+; SI-NEXT: v_readlane_b32 s17, v21, 26
+; SI-NEXT: v_readlane_b32 s10, v23, 57
+; SI-NEXT: v_readlane_b32 s12, v23, 51
+; SI-NEXT: v_readlane_b32 s53, v21, 16
+; SI-NEXT: v_writelane_b32 v22, s46, 25
+; SI-NEXT: s_mov_b32 s93, s41
+; SI-NEXT: v_readlane_b32 s40, v23, 18
+; SI-NEXT: v_readlane_b32 s84, v23, 43
+; SI-NEXT: v_mov_b32_e32 v1, s35
+; SI-NEXT: v_mov_b32_e32 v2, s34
+; SI-NEXT: v_mov_b32_e32 v3, s37
+; SI-NEXT: v_mov_b32_e32 v4, s36
+; SI-NEXT: v_mov_b32_e32 v5, s51
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s65
+; SI-NEXT: v_mov_b32_e32 v10, s99
+; SI-NEXT: v_mov_b32_e32 v13, s5
+; SI-NEXT: v_mov_b32_e32 v14, s83
+; SI-NEXT: v_mov_b32_e32 v16, s81
+; SI-NEXT: s_mov_b32 s20, s64
+; SI-NEXT: v_readlane_b32 s25, v21, 12
+; SI-NEXT: v_readlane_b32 s54, v21, 1
+; SI-NEXT: s_mov_b32 s92, s4
+; SI-NEXT: v_readlane_b32 s64, v21, 9
+; SI-NEXT: v_readlane_b32 s23, v21, 20
+; SI-NEXT: v_readlane_b32 s48, v23, 63
+; SI-NEXT: v_readlane_b32 s4, v21, 3
+; SI-NEXT: v_readlane_b32 s9, v23, 60
+; SI-NEXT: v_readlane_b32 s34, v21, 23
+; SI-NEXT: v_readlane_b32 s11, v23, 58
+; SI-NEXT: v_readlane_b32 s82, v21, 21
+; SI-NEXT: v_readlane_b32 s13, v23, 52
+; SI-NEXT: v_readlane_b32 s98, v23, 49
+; SI-NEXT: v_readlane_b32 s50, v21, 17
+; SI-NEXT: v_readlane_b32 s36, v23, 47
+; SI-NEXT: v_readlane_b32 s96, v23, 53
+; SI-NEXT: v_writelane_b32 v22, s47, 26
+; SI-NEXT: v_readlane_b32 s52, v23, 10
+; SI-NEXT: v_readlane_b32 s80, v23, 12
+; SI-NEXT: v_readlane_b32 s42, v23, 16
+; SI-NEXT: v_readlane_b32 s41, v23, 19
+; SI-NEXT: v_readlane_b32 s16, v23, 20
+; SI-NEXT: v_readlane_b32 s66, v23, 22
+; SI-NEXT: v_readlane_b32 s74, v23, 24
+; SI-NEXT: v_readlane_b32 s85, v23, 44
+; SI-NEXT: v_mov_b32_e32 v6, s57
+; SI-NEXT: v_mov_b32_e32 v11, s89
+; SI-NEXT: v_readlane_b32 s55, v21, 2
+; SI-NEXT: v_readlane_b32 s65, v21, 10
+; SI-NEXT: v_readlane_b32 s49, v21, 0
+; SI-NEXT: v_readlane_b32 s5, v21, 4
+; SI-NEXT: v_readlane_b32 s35, v21, 24
+; SI-NEXT: v_readlane_b32 s83, v21, 22
+; SI-NEXT: v_readlane_b32 s99, v23, 50
+; SI-NEXT: v_readlane_b32 s51, v21, 18
+; SI-NEXT: v_readlane_b32 s37, v23, 48
+; SI-NEXT: v_readlane_b32 s97, v23, 54
+; SI-NEXT: v_readlane_b32 s53, v23, 11
+; SI-NEXT: v_readlane_b32 s81, v23, 13
+; SI-NEXT: v_readlane_b32 s43, v23, 17
+; SI-NEXT: s_mov_b32 s89, s41
+; SI-NEXT: v_readlane_b32 s17, v23, 21
+; SI-NEXT: v_readlane_b32 s67, v23, 23
+; SI-NEXT: v_readlane_b32 s75, v23, 25
+; SI-NEXT: v_readlane_b32 s9, v22, 29
+; SI-NEXT: v_readlane_b32 s11, v22, 30
+; SI-NEXT: v_readlane_b32 s13, v22, 31
+; SI-NEXT: v_readlane_b32 s15, v22, 32
+; SI-NEXT: v_readlane_b32 s16, v22, 33
+; SI-NEXT: v_readlane_b32 s19, v22, 34
+; SI-NEXT: v_readlane_b32 s21, v22, 35
+; SI-NEXT: v_readlane_b32 s23, v22, 36
+; SI-NEXT: v_readlane_b32 s25, v22, 37
+; SI-NEXT: v_readlane_b32 s27, v22, 38
+; SI-NEXT: v_readlane_b32 s40, v22, 39
+; SI-NEXT: v_readlane_b32 s41, v22, 40
+; SI-NEXT: v_readlane_b32 s42, v22, 41
+; SI-NEXT: v_readlane_b32 s45, v22, 42
+; SI-NEXT: v_readlane_b32 s47, v22, 43
+; SI-NEXT: v_readlane_b32 s56, v22, 44
+; SI-NEXT: s_mov_b32 s57, s85
+; SI-NEXT: .LBB91_6: ; %end
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: s_lshl_b32 s6, s36, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s98, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s12, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v17, s5
; SI-NEXT: s_and_b32 s5, s96, 0xff
-; SI-NEXT: s_lshl_b32 s8, s88, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s87, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v56
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 6
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s84, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 7
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 8
-; SI-NEXT: v_readlane_b32 s9, v62, 9
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s84, v62, 10
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s84, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s82, 0xff
-; SI-NEXT: s_lshl_b32 s8, s61, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s51, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v47
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 12
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s70, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 13
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 14
-; SI-NEXT: v_readlane_b32 s9, v62, 15
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 16
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s68, 0xff
-; SI-NEXT: s_lshl_b32 s8, s72, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s67, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v46
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 18
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s64, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 19
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 17
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 20
-; SI-NEXT: v_readlane_b32 s9, v62, 21
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 22
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s54, 0xff
-; SI-NEXT: s_lshl_b32 s8, s57, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s75, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v45
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 24
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s48, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 25
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 23
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 26
-; SI-NEXT: v_readlane_b32 s9, v62, 27
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 28
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, vcc_lo, 0xff
-; SI-NEXT: s_lshl_b32 s8, s58, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s43, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v44
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 30
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s94, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 31
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 29
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 32
-; SI-NEXT: v_readlane_b32 s9, v62, 33
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 34
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s80, 0xff
-; SI-NEXT: s_lshl_b32 s8, s93, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s89, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v43
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 36
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s76, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 37
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 35
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 38
-; SI-NEXT: v_readlane_b32 s9, v62, 39
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 40
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s38, 0xff
-; SI-NEXT: s_lshl_b32 s8, s44, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s37, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v42
+; SI-NEXT: s_lshl_b32 s6, s56, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s57, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 42
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
+; SI-NEXT: v_or_b32_e32 v16, s6, v16
+; SI-NEXT: v_or_b32_e32 v16, s5, v16
; SI-NEXT: s_and_b32 s5, s30, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 43
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 41
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 44
-; SI-NEXT: v_readlane_b32 s9, v62, 45
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 46
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 48
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s8, 0xff
-; SI-NEXT: s_lshl_b32 s8, s27, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s15, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
-; SI-NEXT: v_readlane_b32 s9, v62, 49
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v41
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 50
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s78, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 51
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 52
-; SI-NEXT: v_readlane_b32 s9, v62, 53
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 54
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s6, s10, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s8, 0xff
+; SI-NEXT: v_readlane_b32 s56, v23, 61
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s56, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s86, 0xff
-; SI-NEXT: s_lshl_b32 s8, s28, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s73, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:64
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v40
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 56
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s62, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 57
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s27, v62, 55
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 58
-; SI-NEXT: v_readlane_b32 s9, v62, 59
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 60
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: v_mov_b32_e32 v16, s5
+; SI-NEXT: s_and_b32 s5, s48, 0xff
+; SI-NEXT: s_lshl_b32 s6, s47, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s75, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v15
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:68
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s74, 0xff
-; SI-NEXT: s_lshl_b32 s8, s21, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s59, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_or_b32_e32 v15, s6, v15
+; SI-NEXT: v_readlane_b32 s6, v21, 5
+; SI-NEXT: v_or_b32_e32 v15, s5, v15
+; SI-NEXT: s_and_b32 s5, s54, 0xff
+; SI-NEXT: s_lshl_b32 s4, s4, 8
+; SI-NEXT: v_readlane_b32 s7, v21, 6
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 7
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:12
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v12
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 62
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s46, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 63
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 0
-; SI-NEXT: v_readlane_b32 s9, v61, 1
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 2
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
+; SI-NEXT: v_mov_b32_e32 v15, s4
+; SI-NEXT: s_and_b32 s4, s64, 0xff
+; SI-NEXT: s_lshl_b32 s5, s45, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s67, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 8
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v14, 24, v14
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v14, s5, v14
+; SI-NEXT: v_readlane_b32 s6, v21, 13
+; SI-NEXT: v_or_b32_e32 v14, s4, v14
+; SI-NEXT: s_and_b32 s4, s24, 0xff
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s44, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s50, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:20
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s52, 0xff
-; SI-NEXT: s_lshl_b32 s8, s22, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s45, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: s_and_b32 s4, s22, 0xff
+; SI-NEXT: s_lshl_b32 s5, s42, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v13, s5, v13
+; SI-NEXT: v_or_b32_e32 v13, s4, v13
+; SI-NEXT: s_and_b32 s4, s70, 0xff
+; SI-NEXT: s_lshl_b32 s5, s82, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s34, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s46, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s7, v21, 14
+; SI-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:28
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v9
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 4
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s40, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 5
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s21, v61, 3
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 6
-; SI-NEXT: v_readlane_b32 s9, v61, 7
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 8
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
+; SI-NEXT: v_mov_b32_e32 v13, s4
+; SI-NEXT: s_and_b32 s4, s68, 0xff
+; SI-NEXT: s_lshl_b32 s5, s41, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s89, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 27
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v12
+; SI-NEXT: v_readlane_b32 s7, v21, 28
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v12, s5, v12
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v21, 29
+; SI-NEXT: v_or_b32_e32 v12, s4, v12
+; SI-NEXT: s_and_b32 s4, s86, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 30
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 31
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:36
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s66, 0xff
-; SI-NEXT: s_lshl_b32 s8, s56, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s29, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_mov_b32_e32 v12, s4
+; SI-NEXT: v_readlane_b32 s4, v23, 36
+; SI-NEXT: v_readlane_b32 s5, v23, 37
+; SI-NEXT: v_readlane_b32 s7, v21, 32
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s40, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s43, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 33
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v11
+; SI-NEXT: v_readlane_b32 s7, v21, 34
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v11, s5, v11
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v21, 35
+; SI-NEXT: v_or_b32_e32 v11, s4, v11
+; SI-NEXT: s_and_b32 s4, s38, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 36
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 37
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:44
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v4
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 10
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s24, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 11
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s21, v61, 9
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 12
-; SI-NEXT: v_readlane_b32 s9, v61, 13
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 14
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
+; SI-NEXT: v_mov_b32_e32 v11, s4
+; SI-NEXT: v_readlane_b32 s4, v23, 26
+; SI-NEXT: v_readlane_b32 s5, v23, 27
+; SI-NEXT: v_readlane_b32 s7, v21, 38
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s27, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s93, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 39
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v10
+; SI-NEXT: v_readlane_b32 s7, v21, 40
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v10, s5, v10
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v21, 41
+; SI-NEXT: v_or_b32_e32 v10, s4, v10
+; SI-NEXT: s_and_b32 s4, s20, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 42
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 43
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:52
+; SI-NEXT: v_readlane_b32 s7, v21, 44
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s42, 0xff
-; SI-NEXT: s_lshl_b32 s8, s18, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s23, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_mov_b32_e32 v10, s4
+; SI-NEXT: s_and_b32 s4, s92, 0xff
+; SI-NEXT: s_lshl_b32 s5, s25, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s81, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 45
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT: v_readlane_b32 s7, v21, 46
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v9, s5, v9
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v21, 47
+; SI-NEXT: v_or_b32_e32 v9, s4, v9
+; SI-NEXT: s_and_b32 s4, s18, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 48
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 49
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:60
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v54
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 16
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s16, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 17
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 18
-; SI-NEXT: v_readlane_b32 s9, v61, 19
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s16, v61, 20
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s16, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
+; SI-NEXT: v_mov_b32_e32 v9, s4
+; SI-NEXT: v_readlane_b32 s4, v23, 38
+; SI-NEXT: v_readlane_b32 s5, v23, 39
+; SI-NEXT: v_readlane_b32 s7, v21, 50
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s23, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s53, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 51
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; SI-NEXT: v_readlane_b32 s7, v21, 52
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v8, s5, v8
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v21, 53
+; SI-NEXT: v_or_b32_e32 v8, s4, v8
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 54
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 55
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:68
+; SI-NEXT: v_readlane_b32 s7, v21, 56
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s34, 0xff
-; SI-NEXT: s_lshl_b32 s8, s91, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s19, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: s_and_b32 s4, s26, 0xff
+; SI-NEXT: s_lshl_b32 s5, s21, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s59, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 57
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v7
+; SI-NEXT: v_readlane_b32 s7, v21, 58
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v7, s5, v7
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v21, 59
+; SI-NEXT: v_or_b32_e32 v7, s4, v7
+; SI-NEXT: s_and_b32 s4, s62, 0xff
+; SI-NEXT: v_readlane_b32 s7, v21, 60
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 61
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v21, 62
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s6, v23, 6
+; SI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:76
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v8
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 22
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_and_b32 s5, s10, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 23
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 24
-; SI-NEXT: v_readlane_b32 s9, v61, 25
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v61, 26
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
+; SI-NEXT: v_mov_b32_e32 v7, s4
+; SI-NEXT: s_and_b32 s4, s88, 0xff
+; SI-NEXT: s_lshl_b32 s5, s19, 8
+; SI-NEXT: v_readlane_b32 s7, v23, 7
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s7, 0xff
+; SI-NEXT: v_readlane_b32 s6, v21, 63
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_readlane_b32 s7, v22, 0
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v6, s5, v6
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v22, 1
+; SI-NEXT: v_or_b32_e32 v6, s4, v6
+; SI-NEXT: s_and_b32 s4, s90, 0xff
+; SI-NEXT: v_readlane_b32 s7, v22, 2
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 3
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:84
+; SI-NEXT: v_readlane_b32 s7, v22, 4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s36, 0xff
-; SI-NEXT: s_lshl_b32 s8, s12, 8
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s13, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_readlane_b32 s4, v23, 30
+; SI-NEXT: v_readlane_b32 s5, v23, 31
+; SI-NEXT: v_readlane_b32 s6, v23, 4
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s16, 8
+; SI-NEXT: v_readlane_b32 s7, v23, 5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s7, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 5
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; SI-NEXT: v_readlane_b32 s7, v22, 6
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v5, s5, v5
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v22, 7
+; SI-NEXT: v_or_b32_e32 v5, s4, v5
+; SI-NEXT: s_and_b32 s4, s76, 0xff
+; SI-NEXT: v_readlane_b32 s7, v22, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 9
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:92
+; SI-NEXT: v_readlane_b32 s7, v22, 10
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v11
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 28
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_readlane_b32 s9, v61, 29
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_lshl_b32 s5, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v61, 30
+; SI-NEXT: v_mov_b32_e32 v5, s4
+; SI-NEXT: v_readlane_b32 s4, v23, 28
+; SI-NEXT: v_readlane_b32 s5, v23, 29
+; SI-NEXT: v_readlane_b32 s6, v23, 2
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 31
+; SI-NEXT: s_lshl_b32 s5, s15, 8
+; SI-NEXT: v_readlane_b32 s7, v23, 3
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s7, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 11
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT: v_readlane_b32 s7, v22, 12
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v4, s5, v4
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v22, 13
+; SI-NEXT: v_or_b32_e32 v4, s4, v4
+; SI-NEXT: s_and_b32 s4, s72, 0xff
+; SI-NEXT: v_readlane_b32 s7, v22, 14
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v61, 32
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 15
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s5, s8, s5
+; SI-NEXT: v_readlane_b32 s7, v22, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s6, v23, 0
+; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:100
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: s_and_b32 s4, s60, 0xff
+; SI-NEXT: s_lshl_b32 s5, s13, 8
+; SI-NEXT: v_readlane_b32 s7, v23, 1
; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s7, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 17
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_readlane_b32 s7, v22, 18
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v3, s5, v3
; SI-NEXT: s_lshl_b32 s5, s6, 8
-; SI-NEXT: v_readlane_b32 s6, v61, 34
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
+; SI-NEXT: v_readlane_b32 s6, v22, 19
+; SI-NEXT: v_or_b32_e32 v3, s4, v3
+; SI-NEXT: s_and_b32 s4, s28, 0xff
+; SI-NEXT: v_readlane_b32 s7, v22, 20
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v22, 21
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v22, 22
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s6, v22, 25
+; SI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:108
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: s_and_b32 s4, s14, 0xff
-; SI-NEXT: v_readlane_b32 s7, v61, 35
+; SI-NEXT: v_mov_b32_e32 v3, s4
+; SI-NEXT: s_and_b32 s4, s58, 0xff
+; SI-NEXT: s_lshl_b32 s5, s11, 8
+; SI-NEXT: v_readlane_b32 s7, v22, 26
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s7, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
+; SI-NEXT: v_readlane_b32 s6, v22, 23
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: v_readlane_b32 s7, v22, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v2, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: v_readlane_b32 s6, v23, 34
+; SI-NEXT: v_or_b32_e32 v2, s4, v2
+; SI-NEXT: s_and_b32 s4, s78, 0xff
+; SI-NEXT: v_readlane_b32 s7, v23, 35
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: v_readlane_b32 s6, v23, 32
; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_readlane_b32 s7, v23, 33
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_readlane_b32 s6, v22, 27
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v60
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: s_and_b32 s4, s94, 0xff
+; SI-NEXT: s_lshl_b32 s5, s9, 8
+; SI-NEXT: v_readlane_b32 s7, v22, 28
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s7, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
+; SI-NEXT: v_readlane_b32 s30, v20, 34
+; SI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen
+; SI-NEXT: v_readlane_b32 s57, v23, 62
+; SI-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:56
+; SI-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:64
+; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:72
+; SI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:80
+; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:88
+; SI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:96
+; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:104
+; SI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:112
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s99, v62, 5
-; SI-NEXT: v_readlane_b32 s85, v62, 11
-; SI-NEXT: v_readlane_b32 s30, v63, 34
-; SI-NEXT: v_readlane_b32 s61, v62, 47
-; SI-NEXT: v_readlane_b32 s27, v62, 61
-; SI-NEXT: v_readlane_b32 s21, v61, 15
-; SI-NEXT: v_readlane_b32 s17, v61, 21
-; SI-NEXT: v_readlane_b32 s11, v61, 27
-; SI-NEXT: v_readlane_b32 s9, v61, 33
-; SI-NEXT: v_readlane_b32 s31, v63, 35
-; SI-NEXT: v_readlane_b32 s99, v63, 33
-; SI-NEXT: v_readlane_b32 s98, v63, 32
-; SI-NEXT: v_readlane_b32 s97, v63, 31
-; SI-NEXT: v_readlane_b32 s96, v63, 30
-; SI-NEXT: v_readlane_b32 s87, v63, 29
-; SI-NEXT: v_readlane_b32 s86, v63, 28
-; SI-NEXT: v_readlane_b32 s85, v63, 27
-; SI-NEXT: v_readlane_b32 s84, v63, 26
-; SI-NEXT: v_readlane_b32 s83, v63, 25
-; SI-NEXT: v_readlane_b32 s82, v63, 24
-; SI-NEXT: v_readlane_b32 s81, v63, 23
-; SI-NEXT: v_readlane_b32 s80, v63, 22
-; SI-NEXT: v_readlane_b32 s71, v63, 21
-; SI-NEXT: v_readlane_b32 s70, v63, 20
-; SI-NEXT: v_readlane_b32 s69, v63, 19
-; SI-NEXT: v_readlane_b32 s68, v63, 18
-; SI-NEXT: v_readlane_b32 s67, v63, 17
-; SI-NEXT: v_readlane_b32 s66, v63, 16
-; SI-NEXT: v_readlane_b32 s65, v63, 15
-; SI-NEXT: v_readlane_b32 s64, v63, 14
-; SI-NEXT: v_readlane_b32 s55, v63, 13
-; SI-NEXT: v_readlane_b32 s54, v63, 12
-; SI-NEXT: v_readlane_b32 s53, v63, 11
-; SI-NEXT: v_readlane_b32 s52, v63, 10
-; SI-NEXT: v_readlane_b32 s51, v63, 9
-; SI-NEXT: v_readlane_b32 s50, v63, 8
-; SI-NEXT: v_readlane_b32 s49, v63, 7
-; SI-NEXT: v_readlane_b32 s48, v63, 6
-; SI-NEXT: v_readlane_b32 s39, v63, 5
-; SI-NEXT: v_readlane_b32 s38, v63, 4
-; SI-NEXT: v_readlane_b32 s37, v63, 3
-; SI-NEXT: v_readlane_b32 s36, v63, 2
-; SI-NEXT: v_readlane_b32 s35, v63, 1
-; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_readlane_b32 s31, v20, 35
+; SI-NEXT: v_readlane_b32 s99, v20, 33
+; SI-NEXT: v_readlane_b32 s98, v20, 32
+; SI-NEXT: v_readlane_b32 s97, v20, 31
+; SI-NEXT: v_readlane_b32 s96, v20, 30
+; SI-NEXT: v_readlane_b32 s87, v20, 29
+; SI-NEXT: v_readlane_b32 s86, v20, 28
+; SI-NEXT: v_readlane_b32 s85, v20, 27
+; SI-NEXT: v_readlane_b32 s84, v20, 26
+; SI-NEXT: v_readlane_b32 s83, v20, 25
+; SI-NEXT: v_readlane_b32 s82, v20, 24
+; SI-NEXT: v_readlane_b32 s81, v20, 23
+; SI-NEXT: v_readlane_b32 s80, v20, 22
+; SI-NEXT: v_readlane_b32 s71, v20, 21
+; SI-NEXT: v_readlane_b32 s70, v20, 20
+; SI-NEXT: v_readlane_b32 s69, v20, 19
+; SI-NEXT: v_readlane_b32 s68, v20, 18
+; SI-NEXT: v_readlane_b32 s67, v20, 17
+; SI-NEXT: v_readlane_b32 s66, v20, 16
+; SI-NEXT: v_readlane_b32 s65, v20, 15
+; SI-NEXT: v_readlane_b32 s64, v20, 14
+; SI-NEXT: v_readlane_b32 s55, v20, 13
+; SI-NEXT: v_readlane_b32 s54, v20, 12
+; SI-NEXT: v_readlane_b32 s53, v20, 11
+; SI-NEXT: v_readlane_b32 s52, v20, 10
+; SI-NEXT: v_readlane_b32 s51, v20, 9
+; SI-NEXT: v_readlane_b32 s50, v20, 8
+; SI-NEXT: v_readlane_b32 s49, v20, 7
+; SI-NEXT: v_readlane_b32 s48, v20, 6
+; SI-NEXT: v_readlane_b32 s39, v20, 5
+; SI-NEXT: v_readlane_b32 s38, v20, 4
+; SI-NEXT: v_readlane_b32 s37, v20, 3
+; SI-NEXT: v_readlane_b32 s36, v20, 2
+; SI-NEXT: v_readlane_b32 s35, v20, 1
+; SI-NEXT: v_readlane_b32 s34, v20, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -208150,1147 +204198,1001 @@ define <64 x half> @bitcast_v64bf16_to_v64f16(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:4
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v29
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v3
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v28
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v27
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v5
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v2
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v46
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v55
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v3
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v47
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v42
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v19
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v30
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v26
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v23
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v57
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v43
+; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v36
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v58
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v45
+; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v54
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v55
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v59
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v8
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v20
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v9
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v42
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v51
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v10
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v18
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v11
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v50
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v49
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v12
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v13
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v48
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v11
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v39
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v14
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v38
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v13
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v37
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v16
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v17
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v30
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v15
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v45
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v19
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v17
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v42
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v21
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v60
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v36
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB100_2
+; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v19
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v55
+; SI-NEXT: v_or_b32_e32 v34, v1, v32
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v59
+; SI-NEXT: v_or_b32_e32 v33, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v42
+; SI-NEXT: v_or_b32_e32 v32, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v51
+; SI-NEXT: v_or_b32_e32 v5, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v50
+; SI-NEXT: v_or_b32_e32 v3, v1, v3
+; SI-NEXT: v_alignbit_b32 v1, v34, v44, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v55
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v49
+; SI-NEXT: v_or_b32_e32 v11, v1, v7
+; SI-NEXT: v_alignbit_b32 v1, v33, v31, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v48
+; SI-NEXT: v_or_b32_e32 v9, v1, v7
+; SI-NEXT: v_alignbit_b32 v1, v32, v63, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v52
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v25
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v39
+; SI-NEXT: v_or_b32_e32 v7, v1, v7
+; SI-NEXT: v_alignbit_b32 v1, v5, v56, 16
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v53
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v25
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v38
+; SI-NEXT: v_or_b32_e32 v5, v1, v5
+; SI-NEXT: v_alignbit_b32 v1, v3, v41, 16
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v49
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v37
+; SI-NEXT: v_or_b32_e32 v3, v1, v3
+; SI-NEXT: v_alignbit_b32 v1, v11, v40, 16
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v50
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v30
+; SI-NEXT: v_or_b32_e32 v11, v1, v11
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v9, v53, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v38
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v39
-; SI-NEXT: ; kill: killed $vgpr21
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v23
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v44
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; kill: killed $vgpr32
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_or_b32_e32 v9, v1, v9
+; SI-NEXT: v_alignbit_b32 v1, v7, v52, 16
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v35
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v36
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v25
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v34
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v37
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v9, v9, v57, 16
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_or_b32_e32 v7, v1, v7
+; SI-NEXT: v_alignbit_b32 v1, v5, v62, 16
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v57
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v43
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v7, v47, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v31
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v31
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; kill: killed $vgpr31
; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v62
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v59
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v46
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v51
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v33
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v32
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v63
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v61
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v60
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v47
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v44
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v41
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; kill: killed $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; kill: killed $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; kill: killed $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; kill: killed $vgpr31
; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; kill: killed $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; kill: killed $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; kill: killed $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; kill: killed $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; kill: killed $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB100_2
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v30
-; SI-NEXT: v_alignbit_b32 v2, v2, v45, 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v28
-; SI-NEXT: v_alignbit_b32 v2, v2, v7, 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v26
-; SI-NEXT: v_alignbit_b32 v2, v2, v43, 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v24
-; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v22
-; SI-NEXT: v_alignbit_b32 v1, v1, v40, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: v_alignbit_b32 v1, v1, v54, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v18
-; SI-NEXT: v_alignbit_b32 v1, v1, v11, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
-; SI-NEXT: v_alignbit_b32 v1, v1, v50, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
-; SI-NEXT: v_alignbit_b32 v1, v1, v3, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
-; SI-NEXT: v_alignbit_b32 v1, v1, v48, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_or_b32_e32 v5, v1, v5
+; SI-NEXT: v_alignbit_b32 v1, v3, v61, 16
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v5, v46, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v10
-; SI-NEXT: v_alignbit_b32 v1, v1, v19, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v29
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_alignbit_b32 v1, v1, v9, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v63
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_or_b32_e32 v3, v1, v3
+; SI-NEXT: v_alignbit_b32 v1, v11, v58, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v3, v45, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
-; SI-NEXT: v_alignbit_b32 v1, v1, v17, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v60
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v62
-; SI-NEXT: v_alignbit_b32 v1, v1, v5, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v54
+; SI-NEXT: v_or_b32_e32 v0, v0, v56
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v59
-; SI-NEXT: v_alignbit_b32 v1, v1, v15, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v1, v43, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v57
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v41
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v6, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v30
-; SI-NEXT: v_alignbit_b32 v0, v6, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v0, v0, v40
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v4, v63, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v28
-; SI-NEXT: v_alignbit_b32 v0, v4, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v0, v0, v53
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v61, v60, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v26
-; SI-NEXT: v_alignbit_b32 v0, v61, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v0, v0, v52
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v58, v56, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
-; SI-NEXT: v_alignbit_b32 v0, v58, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v62
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v47, v46, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v22
-; SI-NEXT: v_alignbit_b32 v0, v47, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v0, v0, v61
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v44, v42, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v20
-; SI-NEXT: v_alignbit_b32 v0, v44, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v0, v0, v58
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v41, v55, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v18
-; SI-NEXT: v_alignbit_b32 v0, v41, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v0, v0, v57
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v53, v52, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v16
-; SI-NEXT: v_alignbit_b32 v0, v53, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v0, v0, v47
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v51, v49, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
-; SI-NEXT: v_alignbit_b32 v0, v51, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v0, v0, v46
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v39, v38, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v12
-; SI-NEXT: v_alignbit_b32 v0, v39, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v0, v0, v45
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v37, v21, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
-; SI-NEXT: v_alignbit_b32 v0, v37, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v36, v23, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: v_alignbit_b32 v0, v36, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v35, v25, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
-; SI-NEXT: v_alignbit_b32 v0, v35, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v34, v27, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v62
-; SI-NEXT: v_alignbit_b32 v0, v34, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr52
; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v33, v29, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v59
-; SI-NEXT: v_alignbit_b32 v0, v33, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_alignbit_b32 v32, v31, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
-; SI-NEXT: v_alignbit_b32 v0, v32, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr58
; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v43
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr43
; SI-NEXT: .LBB100_2: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_xor_b64 exec, exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v30
-; SI-NEXT: v_alignbit_b32 v2, v4, v2, 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v44
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_alignbit_b32 v30, v33, v32, 16
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_alignbit_b32 v4, v6, v4, 16
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v30, v31, v32, 16
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v63
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v30, v32, v31, 16
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v56
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v30, v32, v31, 16
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v41
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v8
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v6
-; SI-NEXT: v_alignbit_b32 v7, v21, v7, 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v30, v32, v31, 16
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v7
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v12
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v18
-; SI-NEXT: v_alignbit_b32 v11, v20, v11, 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v14
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v16
-; SI-NEXT: v_alignbit_b32 v11, v20, v11, 16
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v62
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v16
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v11
-; SI-NEXT: v_alignbit_b32 v3, v14, v3, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v18
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v61
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v18
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; SI-NEXT: v_alignbit_b32 v3, v14, v3, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v58
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v20
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v19
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v57
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
-; SI-NEXT: v_alignbit_b32 v3, v14, v3, 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v22
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_alignbit_b32 v3, v9, v3, 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v13
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v47
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v24
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v9
-; SI-NEXT: v_alignbit_b32 v3, v13, v3, 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v46
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v26
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; SI-NEXT: v_alignbit_b32 v3, v13, v3, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v59
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v28
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v45
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_alignbit_b32 v3, v14, v3, 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v57
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v43
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v14, v0, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_alignbit_b32 v32, v31, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: v_alignbit_b32 v0, v32, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_alignbit_b32 v33, v29, v3, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v34, v27, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v35, v25, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v36, v23, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v37, v21, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v39, v38, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v51, v49, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_alignbit_b32 v53, v52, v13, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_alignbit_b32 v44, v42, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_alignbit_b32 v47, v46, v15, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_alignbit_b32 v58, v56, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_alignbit_b32 v1, v58, v1, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v47, v15, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v44, v7, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_alignbit_b32 v61, v60, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_alignbit_b32 v41, v55, v14, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v18
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v41, v14, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v53, v13, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v51, v11, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v39, v12, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v37, v10, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v36, v8, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v35, v9, 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v30
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v60
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v54
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v32
+; SI-NEXT: v_alignbit_b32 v52, v36, v10, 16
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_alignbit_b32 v35, v30, v29, 16
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v29
+; SI-NEXT: v_alignbit_b32 v29, v32, v27, 16
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v27
+; SI-NEXT: v_alignbit_b32 v27, v33, v25, 16
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v25
+; SI-NEXT: v_alignbit_b32 v25, v34, v23, 16
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v34, v5, 16
-; SI-NEXT: v_alignbit_b32 v16, v61, v16, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v23
+; SI-NEXT: v_alignbit_b32 v23, v30, v21, 16
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v21
+; SI-NEXT: v_alignbit_b32 v21, v37, v19, 16
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v19
+; SI-NEXT: v_alignbit_b32 v19, v38, v17, 16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v17
+; SI-NEXT: v_alignbit_b32 v17, v39, v15, 16
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v15
+; SI-NEXT: v_alignbit_b32 v15, v48, v13, 16
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v53, v49, v11, 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v13
+; SI-NEXT: v_alignbit_b32 v13, v50, v9, 16
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v9
+; SI-NEXT: v_alignbit_b32 v9, v51, v7, 16
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v7
+; SI-NEXT: v_alignbit_b32 v7, v42, v5, 16
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v5
+; SI-NEXT: v_alignbit_b32 v5, v59, v3, 16
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v3
+; SI-NEXT: v_alignbit_b32 v1, v55, v1, 16
+; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v5, v2, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v7, v4, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v9, v6, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v13, v8, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v53, v11, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v15, v12, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v17, v14, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v19, v16, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v21, v18, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v23, v20, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v25, v22, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v27, v24, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v29, v26, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v35, v28, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v35, v52
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v33, v3, 16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_alignbit_b32 v4, v63, v4, 16
-; SI-NEXT: v_alignbit_b32 v17, v4, v17, 16
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v6, v2, v6, 16
-; SI-NEXT: v_alignbit_b32 v18, v6, v18, 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v52, v10, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; SI-NEXT: .LBB100_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v56
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v46
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v42
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v55
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v52
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v49
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v38
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_and_b32_e32 v32, 0xffff, v32
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_or_b32_e32 v31, v32, v31
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v55
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v59
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v42
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v51
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v50
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v49
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v48
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v38
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v37
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v34
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v33
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v32
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v36
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v60
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: s_waitcnt vmcnt(13)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: s_waitcnt vmcnt(10)
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v63
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_or_b32_e32 v3, v3, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v61
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: v_or_b32_e32 v6, v6, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v58
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
; SI-NEXT: v_or_b32_e32 v7, v7, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v47
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v44
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
; SI-NEXT: v_or_b32_e32 v12, v12, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v41
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: v_or_b32_e32 v14, v14, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v53
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
; SI-NEXT: v_or_b32_e32 v16, v16, v17
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v51
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
; SI-NEXT: v_or_b32_e32 v17, v17, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
; SI-NEXT: v_or_b32_e32 v18, v18, v19
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v39
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
; SI-NEXT: v_or_b32_e32 v19, v19, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: v_or_b32_e32 v20, v20, v22
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v37
-; SI-NEXT: v_or_b32_e32 v21, v22, v21
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT: v_or_b32_e32 v22, v22, v24
-; SI-NEXT: v_and_b32_e32 v24, 0xffff, v36
-; SI-NEXT: v_or_b32_e32 v23, v24, v23
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT: v_or_b32_e32 v24, v24, v26
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v35
-; SI-NEXT: v_or_b32_e32 v25, v26, v25
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT: v_or_b32_e32 v26, v26, v28
-; SI-NEXT: v_and_b32_e32 v28, 0xffff, v34
-; SI-NEXT: v_or_b32_e32 v27, v28, v27
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; SI-NEXT: v_or_b32_e32 v28, v28, v30
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v33
-; SI-NEXT: v_or_b32_e32 v29, v30, v29
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_or_b32_e32 v30, v30, v33
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v35
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64bf16_to_v64f16:
@@ -211610,1682 +207512,1192 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: v_writelane_b32 v63, s34, 0
; SI-NEXT: v_writelane_b32 v63, s35, 1
; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s30, 3
-; SI-NEXT: v_writelane_b32 v63, s31, 4
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s60, v12
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s17
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s19
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: v_readfirstlane_b32 s78, v7
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: v_readfirstlane_b32 s62, v11
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: v_readfirstlane_b32 s72, v10
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s21
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s92, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v18
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s79
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s75
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s59
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s57
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s42
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s23
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s30
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s94
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s90
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s78
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s62
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v63, s37, 3
+; SI-NEXT: v_writelane_b32 v63, s38, 4
+; SI-NEXT: v_writelane_b32 v63, s39, 5
+; SI-NEXT: v_writelane_b32 v63, s48, 6
+; SI-NEXT: v_writelane_b32 v63, s49, 7
+; SI-NEXT: v_writelane_b32 v63, s50, 8
+; SI-NEXT: v_writelane_b32 v63, s51, 9
+; SI-NEXT: v_writelane_b32 v63, s52, 10
+; SI-NEXT: v_writelane_b32 v63, s53, 11
+; SI-NEXT: v_writelane_b32 v63, s54, 12
+; SI-NEXT: v_writelane_b32 v63, s55, 13
+; SI-NEXT: v_writelane_b32 v63, s64, 14
+; SI-NEXT: v_writelane_b32 v63, s65, 15
+; SI-NEXT: v_writelane_b32 v63, s66, 16
+; SI-NEXT: v_writelane_b32 v63, s67, 17
+; SI-NEXT: v_writelane_b32 v63, s68, 18
+; SI-NEXT: v_writelane_b32 v63, s69, 19
+; SI-NEXT: v_writelane_b32 v63, s70, 20
+; SI-NEXT: v_writelane_b32 v63, s71, 21
+; SI-NEXT: v_writelane_b32 v63, s80, 22
+; SI-NEXT: v_writelane_b32 v63, s81, 23
+; SI-NEXT: v_writelane_b32 v63, s82, 24
+; SI-NEXT: v_writelane_b32 v63, s83, 25
+; SI-NEXT: v_writelane_b32 v63, s84, 26
+; SI-NEXT: v_writelane_b32 v63, s85, 27
+; SI-NEXT: v_writelane_b32 v63, s86, 28
+; SI-NEXT: v_writelane_b32 v63, s87, 29
+; SI-NEXT: v_writelane_b32 v63, s96, 30
+; SI-NEXT: v_writelane_b32 v63, s97, 31
+; SI-NEXT: v_writelane_b32 v63, s98, 32
+; SI-NEXT: v_writelane_b32 v63, s99, 33
+; SI-NEXT: v_writelane_b32 v63, s30, 34
+; SI-NEXT: v_writelane_b32 v63, s31, 35
+; SI-NEXT: s_lshr_b32 s4, s16, 16
+; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
+; SI-NEXT: s_mov_b32 s8, s28
+; SI-NEXT: v_writelane_b32 v62, s4, 0
+; SI-NEXT: v_readfirstlane_b32 s4, v16
+; SI-NEXT: s_mov_b32 s31, s4
+; SI-NEXT: s_lshr_b32 s4, s4, 16
+; SI-NEXT: s_mov_b32 s28, s19
+; SI-NEXT: s_mov_b32 s74, s17
+; SI-NEXT: s_lshr_b32 s45, s19, 16
+; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: v_readfirstlane_b32 s70, v17
+; SI-NEXT: v_writelane_b32 v62, s4, 1
+; SI-NEXT: v_readfirstlane_b32 s55, v15
+; SI-NEXT: v_readfirstlane_b32 s4, v14
+; SI-NEXT: v_readfirstlane_b32 s53, v13
+; SI-NEXT: v_readfirstlane_b32 s88, v12
+; SI-NEXT: v_readfirstlane_b32 s51, v11
+; SI-NEXT: v_readfirstlane_b32 s83, v10
+; SI-NEXT: v_readfirstlane_b32 s59, v9
+; SI-NEXT: v_readfirstlane_b32 s81, v8
+; SI-NEXT: v_readfirstlane_b32 s58, v7
+; SI-NEXT: v_readfirstlane_b32 s61, v6
+; SI-NEXT: v_readfirstlane_b32 s37, v5
+; SI-NEXT: v_readfirstlane_b32 s19, v4
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v3
+; SI-NEXT: v_readfirstlane_b32 s99, v2
+; SI-NEXT: v_readfirstlane_b32 s17, v1
+; SI-NEXT: v_readfirstlane_b32 s82, v0
+; SI-NEXT: s_mov_b32 s10, s26
+; SI-NEXT: s_mov_b32 s6, s25
+; SI-NEXT: s_mov_b32 s79, s21
+; SI-NEXT: s_lshr_b32 s54, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s8, 16
+; SI-NEXT: s_lshr_b32 s52, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s26, 16
+; SI-NEXT: s_lshr_b32 s50, s25, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s24, 16
+; SI-NEXT: s_lshr_b32 s75, s23, 16
+; SI-NEXT: s_lshr_b32 s78, s22, 16
+; SI-NEXT: s_lshr_b32 s73, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s68, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s70, 16
+; SI-NEXT: s_lshr_b32 s92, s55, 16
+; SI-NEXT: s_lshr_b32 s26, s4, 16
+; SI-NEXT: s_lshr_b32 s76, s53, 16
+; SI-NEXT: s_lshr_b32 s71, s88, 16
+; SI-NEXT: s_lshr_b32 s60, s51, 16
+; SI-NEXT: s_lshr_b32 s62, s83, 16
+; SI-NEXT: s_lshr_b32 s49, s59, 16
+; SI-NEXT: s_lshr_b32 s77, s81, 16
+; SI-NEXT: s_lshr_b32 s39, s58, 16
+; SI-NEXT: s_lshr_b32 s21, s61, 16
+; SI-NEXT: s_lshr_b32 s35, s37, 16
+; SI-NEXT: s_lshr_b32 s65, s19, 16
+; SI-NEXT: s_lshr_b32 s25, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s98, s99, 16
+; SI-NEXT: s_lshr_b32 s64, s17, 16
+; SI-NEXT: s_lshr_b32 s36, s82, 16
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: s_mov_b32 s89, s29
+; SI-NEXT: s_mov_b32 s93, s24
+; SI-NEXT: s_cmp_lg_u32 s5, 0
+; SI-NEXT: v_writelane_b32 v62, s26, 2
; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_mov_b32_e32 v46, v22
-; SI-NEXT: v_mov_b32_e32 v22, v13
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v5, v14
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v41, v16
-; SI-NEXT: v_mov_b32_e32 v43, v49
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; SI-NEXT: v_mov_b32_e32 v39, v17
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v48
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v4
-; SI-NEXT: v_mov_b32_e32 v24, v36
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v62
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v9
-; SI-NEXT: v_mov_b32_e32 v49, v9
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[51:52], v[0:1], 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v52, v51
-; SI-NEXT: v_mov_b32_e32 v51, v50
-; SI-NEXT: v_mov_b32_e32 v50, v4
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshr_b64 v[13:14], v[0:1], 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_writelane_b32 v62, s4, 54
+; SI-NEXT: v_writelane_b32 v62, s71, 55
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s80, s4, 16
+; SI-NEXT: s_or_b32 s5, s5, s80
+; SI-NEXT: v_writelane_b32 v62, s5, 53
+; SI-NEXT: v_writelane_b32 v62, s68, 56
+; SI-NEXT: v_readlane_b32 s4, v62, 56
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s68, s4, 16
+; SI-NEXT: s_or_b32 s5, s5, s68
+; SI-NEXT: v_writelane_b32 v62, s5, 52
+; SI-NEXT: v_writelane_b32 v62, s69, 57
+; SI-NEXT: v_readlane_b32 s4, v62, 57
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s4, 16
+; SI-NEXT: s_or_b32 s5, s5, s56
+; SI-NEXT: v_writelane_b32 v62, s5, 51
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s94, s78, 16
+; SI-NEXT: s_or_b32 s5, s5, s94
+; SI-NEXT: v_writelane_b32 v62, s5, 50
+; SI-NEXT: s_and_b32 s5, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s44, vcc_lo, 16
+; SI-NEXT: s_or_b32 s5, s5, s44
+; SI-NEXT: v_writelane_b32 v62, s5, 49
+; SI-NEXT: s_and_b32 s5, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s34, 16
+; SI-NEXT: v_writelane_b32 v62, s16, 58
+; SI-NEXT: s_or_b32 s5, s5, s46
+; SI-NEXT: v_writelane_b32 v62, s5, 48
+; SI-NEXT: s_and_b32 s5, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s38, 16
+; SI-NEXT: s_or_b32 s5, s5, s40
+; SI-NEXT: v_writelane_b32 v62, s5, 47
+; SI-NEXT: s_and_b32 s5, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s84, s36, 16
+; SI-NEXT: s_or_b32 s5, s5, s84
+; SI-NEXT: v_writelane_b32 v62, s5, 46
+; SI-NEXT: s_and_b32 s5, s74, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s48, 16
+; SI-NEXT: s_mov_b32 s71, s62
+; SI-NEXT: s_mov_b32 s62, s81
+; SI-NEXT: s_or_b32 s81, s5, s7
+; SI-NEXT: s_and_b32 s7, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s45, 16
+; SI-NEXT: s_or_b32 s69, s7, s9
+; SI-NEXT: s_and_b32 s9, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s73, 16
+; SI-NEXT: s_or_b32 s57, s9, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s75, 16
+; SI-NEXT: s_or_b32 s95, s11, s13
+; SI-NEXT: s_and_b32 s13, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s50, 16
+; SI-NEXT: s_mov_b32 s16, s22
+; SI-NEXT: s_mov_b32 s63, s45
+; SI-NEXT: s_or_b32 s45, s13, s15
+; SI-NEXT: s_and_b32 s15, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s52, 16
+; SI-NEXT: s_or_b32 s47, s15, s22
+; SI-NEXT: s_and_b32 s15, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s54, 16
+; SI-NEXT: s_or_b32 s41, s15, s7
+; SI-NEXT: s_and_b32 s15, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s64, 16
+; SI-NEXT: s_and_b32 s14, s99, 0xffff
+; SI-NEXT: s_or_b32 s85, s15, s7
+; SI-NEXT: s_and_b32 s15, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s25, 16
+; SI-NEXT: s_lshl_b32 s90, s98, 16
+; SI-NEXT: s_or_b32 s91, s15, s7
+; SI-NEXT: s_or_b32 s4, s14, s90
+; SI-NEXT: s_and_b32 s14, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s35, 16
+; SI-NEXT: s_or_b32 s87, s14, s15
+; SI-NEXT: s_lshl_b32 s86, s65, 16
+; SI-NEXT: s_and_b32 s14, s19, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 45
+; SI-NEXT: s_or_b32 s4, s14, s86
+; SI-NEXT: s_and_b32 s14, s58, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s39, 16
+; SI-NEXT: s_or_b32 s97, s14, s15
+; SI-NEXT: s_lshl_b32 s96, s21, 16
+; SI-NEXT: s_and_b32 s14, s61, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 44
+; SI-NEXT: s_or_b32 s4, s14, s96
+; SI-NEXT: s_and_b32 s14, s59, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s49, 16
+; SI-NEXT: s_or_b32 s11, s14, s15
+; SI-NEXT: s_mov_b32 s9, s10
+; SI-NEXT: s_lshl_b32 s10, s77, 16
+; SI-NEXT: s_and_b32 s14, s62, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 43
+; SI-NEXT: s_or_b32 s4, s14, s10
+; SI-NEXT: s_and_b32 s14, s51, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s60, 16
+; SI-NEXT: s_or_b32 s15, s14, s15
+; SI-NEXT: s_lshl_b32 s14, s71, 16
+; SI-NEXT: s_and_b32 s13, s83, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 42
+; SI-NEXT: s_or_b32 s4, s13, s14
+; SI-NEXT: v_writelane_b32 v62, s4, 41
+; SI-NEXT: v_readlane_b32 s4, v62, 55
+; SI-NEXT: s_mov_b32 s29, s6
+; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_and_b32 s24, s88, 0xffff
+; SI-NEXT: s_or_b32 s4, s24, s6
+; SI-NEXT: v_writelane_b32 v62, s4, 40
+; SI-NEXT: v_readlane_b32 s4, v62, 54
+; SI-NEXT: s_mov_b32 s66, s17
+; SI-NEXT: s_lshl_b32 s42, s26, 16
+; SI-NEXT: s_and_b32 s17, s4, 0xffff
+; SI-NEXT: s_or_b32 s4, s17, s42
+; SI-NEXT: s_mov_b32 s67, s25
+; SI-NEXT: s_and_b32 s24, s55, 0xffff
+; SI-NEXT: s_lshl_b32 s25, s92, 16
+; SI-NEXT: v_writelane_b32 v62, s4, 39
+; SI-NEXT: s_mov_b32 s17, s81
+; SI-NEXT: s_and_b32 s13, s53, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s76, 16
+; SI-NEXT: s_or_b32 s43, s24, s25
+; SI-NEXT: s_and_b32 s24, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s25, s30, 16
+; SI-NEXT: v_readlane_b32 s4, v62, 1
+; SI-NEXT: v_writelane_b32 v62, s16, 5
+; SI-NEXT: s_or_b32 s7, s13, s12
+; SI-NEXT: s_or_b32 s13, s24, s25
+; SI-NEXT: v_writelane_b32 v62, s17, 6
+; SI-NEXT: s_lshr_b64 s[24:25], s[80:81], 16
+; SI-NEXT: v_writelane_b32 v62, s24, 3
+; SI-NEXT: v_writelane_b32 v62, s25, 4
+; SI-NEXT: s_mov_b32 s17, s69
+; SI-NEXT: s_lshl_b32 s12, s4, 16
+; SI-NEXT: s_mov_b32 s81, s62
+; SI-NEXT: s_mov_b32 s62, s71
+; SI-NEXT: v_readlane_b32 s71, v62, 55
+; SI-NEXT: v_readlane_b32 s4, v62, 54
+; SI-NEXT: v_writelane_b32 v62, s16, 9
+; SI-NEXT: v_writelane_b32 v62, s17, 10
+; SI-NEXT: s_lshr_b64 s[24:25], s[68:69], 16
+; SI-NEXT: v_writelane_b32 v62, s24, 7
+; SI-NEXT: v_writelane_b32 v62, s25, 8
+; SI-NEXT: s_mov_b32 s17, s57
+; SI-NEXT: v_readlane_b32 s68, v62, 56
+; SI-NEXT: v_writelane_b32 v62, s16, 13
+; SI-NEXT: v_writelane_b32 v62, s17, 14
+; SI-NEXT: s_lshr_b64 s[24:25], s[56:57], 16
+; SI-NEXT: v_writelane_b32 v62, s24, 11
+; SI-NEXT: v_writelane_b32 v62, s25, 12
+; SI-NEXT: s_mov_b32 s17, s95
+; SI-NEXT: v_readlane_b32 s69, v62, 57
+; SI-NEXT: v_writelane_b32 v62, s16, 17
+; SI-NEXT: v_writelane_b32 v62, s17, 18
+; SI-NEXT: s_lshr_b64 s[24:25], s[94:95], 16
+; SI-NEXT: v_writelane_b32 v62, s24, 15
+; SI-NEXT: v_writelane_b32 v62, s25, 16
+; SI-NEXT: s_mov_b32 s17, s45
+; SI-NEXT: v_writelane_b32 v62, s16, 21
+; SI-NEXT: v_writelane_b32 v62, s17, 22
+; SI-NEXT: s_lshr_b64 s[24:25], s[44:45], 16
+; SI-NEXT: v_writelane_b32 v62, s24, 19
+; SI-NEXT: v_writelane_b32 v62, s25, 20
+; SI-NEXT: s_lshr_b64 s[24:25], s[46:47], 16
+; SI-NEXT: v_writelane_b32 v62, s24, 23
+; SI-NEXT: v_writelane_b32 v62, s25, 24
+; SI-NEXT: s_mov_b32 s22, s16
+; SI-NEXT: s_lshr_b64 s[16:17], s[84:85], 16
+; SI-NEXT: v_writelane_b32 v62, s16, 25
+; SI-NEXT: v_writelane_b32 v62, s17, 26
+; SI-NEXT: s_lshr_b64 s[16:17], s[90:91], 16
+; SI-NEXT: v_writelane_b32 v62, s16, 27
+; SI-NEXT: v_writelane_b32 v62, s17, 28
+; SI-NEXT: s_lshr_b64 s[16:17], s[86:87], 16
+; SI-NEXT: v_writelane_b32 v62, s16, 29
+; SI-NEXT: v_writelane_b32 v62, s17, 30
+; SI-NEXT: s_lshr_b64 s[16:17], s[96:97], 16
+; SI-NEXT: v_writelane_b32 v62, s16, 31
+; SI-NEXT: s_mov_b64 s[94:95], s[40:41]
+; SI-NEXT: s_mov_b32 s41, s47
+; SI-NEXT: v_writelane_b32 v62, s17, 32
+; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
+; SI-NEXT: v_writelane_b32 v62, s46, 33
+; SI-NEXT: v_writelane_b32 v62, s47, 34
+; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 16
+; SI-NEXT: v_writelane_b32 v62, s46, 35
+; SI-NEXT: v_writelane_b32 v62, s47, 36
+; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
+; SI-NEXT: s_and_b32 s72, s31, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s46, 37
+; SI-NEXT: s_mov_b32 s5, s31
+; SI-NEXT: s_or_b32 s31, s72, s12
+; SI-NEXT: s_lshr_b64 s[44:45], s[94:95], 16
+; SI-NEXT: v_writelane_b32 v62, s47, 38
+; SI-NEXT: s_mov_b32 s45, s63
+; SI-NEXT: s_mov_b32 s25, s67
+; SI-NEXT: s_mov_b32 s17, s66
+; SI-NEXT: s_mov_b32 s10, s9
+; SI-NEXT: s_mov_b32 s14, s31
+; SI-NEXT: s_mov_b32 s6, s29
+; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 16
+; SI-NEXT: v_readlane_b32 s16, v62, 58
+; SI-NEXT: s_mov_b32 s31, s5
+; SI-NEXT: s_lshr_b64 s[56:57], s[12:13], 16
+; SI-NEXT: s_mov_b64 s[66:67], 0
+; SI-NEXT: s_branch .LBB101_3
+; SI-NEXT: .LBB101_2:
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; kill: killed $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; kill: killed $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; kill: killed $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: s_mov_b64 s[66:67], -1
+; SI-NEXT: v_writelane_b32 v62, s14, 3
+; SI-NEXT: v_writelane_b32 v62, s15, 4
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; kill: killed $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $sgpr97
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: v_writelane_b32 v62, s4, 5
+; SI-NEXT: v_writelane_b32 v62, s5, 6
+; SI-NEXT: v_writelane_b32 v62, s14, 7
+; SI-NEXT: v_writelane_b32 v62, s15, 8
+; SI-NEXT: v_writelane_b32 v62, s6, 9
+; SI-NEXT: v_writelane_b32 v62, s7, 10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: v_writelane_b32 v62, s14, 11
+; SI-NEXT: v_writelane_b32 v62, s15, 12
+; SI-NEXT: v_writelane_b32 v62, s6, 13
+; SI-NEXT: v_writelane_b32 v62, s7, 14
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: v_writelane_b32 v62, s14, 15
+; SI-NEXT: v_writelane_b32 v62, s15, 16
+; SI-NEXT: v_writelane_b32 v62, s8, 17
+; SI-NEXT: v_writelane_b32 v62, s9, 18
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: v_writelane_b32 v62, s14, 19
+; SI-NEXT: v_writelane_b32 v62, s15, 20
+; SI-NEXT: v_writelane_b32 v62, s10, 21
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s11, 22
+; SI-NEXT: ; kill: killed $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: v_writelane_b32 v62, s14, 23
+; SI-NEXT: v_writelane_b32 v62, s15, 24
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; kill: killed $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; kill: killed $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s14, 25
+; SI-NEXT: v_writelane_b32 v62, s15, 26
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s14, 27
+; SI-NEXT: v_writelane_b32 v62, s15, 28
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s14, 29
+; SI-NEXT: v_writelane_b32 v62, s15, 30
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s14, 31
+; SI-NEXT: v_writelane_b32 v62, s15, 32
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s14, 33
+; SI-NEXT: v_writelane_b32 v62, s15, 34
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v62, s14, 35
+; SI-NEXT: v_writelane_b32 v62, s15, 36
+; SI-NEXT: v_writelane_b32 v62, s26, 37
+; SI-NEXT: v_writelane_b32 v62, s27, 38
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: .LBB101_3: ; %Flow
+; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB101_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: v_mov_b32_e32 v23, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: v_readlane_b32 s5, v62, 0
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: v_add_f32_e32 v36, s5, v23
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v36
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v21
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[13:14], v[0:1], 16
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v13, v52
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v42
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v10
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mov_b32_e32 v15, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v56
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v56
-; SI-NEXT: v_lshr_b64 v[40:41], v[15:16], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s18, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v38
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: v_add_f32_e32 v24, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v27
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v25
-; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v34
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v60
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v8, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[19:20], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v29
-; SI-NEXT: v_mov_b32_e32 v9, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v32
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s20, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: v_add_f32_e32 v21, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v33
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[19:20], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v21
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s22, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: v_add_f32_e32 v19, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v54
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[15:16], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v19
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v16, v35
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
-; SI-NEXT: v_mov_b32_e32 v2, v0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v0, v26
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; SI-NEXT: v_add_f32_e32 v17, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mov_b32_e32 v37, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v31
-; SI-NEXT: v_lshr_b64 v[11:12], v[33:34], 16
-; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v12, v51
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshr_b64 v[32:33], v[44:45], 16
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v50
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v47
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[28:29], v[37:38], 16
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v49
-; SI-NEXT: v_mov_b32_e32 v49, v40
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mov_b32_e32 v23, v52
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[35:36], v[3:4], 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v46
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: v_add_f32_e32 v15, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v48
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v15
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s38, 16
+; SI-NEXT: v_add_f32_e32 v13, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v21
-; SI-NEXT: v_mov_b32_e32 v36, v22
-; SI-NEXT: v_mov_b32_e32 v48, v39
-; SI-NEXT: v_mov_b32_e32 v22, v51
-; SI-NEXT: v_mov_b32_e32 v46, v43
-; SI-NEXT: v_lshr_b64 v[53:54], v[22:23], 16
-; SI-NEXT: v_mov_b32_e32 v54, v15
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s36, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v62
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v8
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s19, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s65, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v42
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s61, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v30
-; SI-NEXT: v_lshr_b64 v[29:30], v[12:13], 16
-; SI-NEXT: v_mov_b32_e32 v30, v35
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_mov_b32_e32 v0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v0
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v1, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s77, 16
+; SI-NEXT: v_add_f32_e32 v0, s5, v23
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v47
-; SI-NEXT: v_mov_b32_e32 v47, v44
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s5, s88, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s5, v23
+; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: v_add_f32_e32 v4, s5, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_readlane_b32 s4, v62, 2
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v52, v23
-; SI-NEXT: v_mov_b32_e32 v45, v4
-; SI-NEXT: v_mov_b32_e32 v44, v3
-; SI-NEXT: v_lshr_b64 v[3:4], v[50:51], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s31, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v49
-; SI-NEXT: v_mov_b32_e32 v43, v45
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[48:49], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_readlane_b32 s4, v62, 1
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[44:45], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s70, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s30, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mov_b32_e32 v50, v19
-; SI-NEXT: v_lshr_b64 v[0:1], v[1:2], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v11
-; SI-NEXT: v_lshr_b64 v[48:49], v[48:49], 16
-; SI-NEXT: v_mov_b32_e32 v49, v46
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mov_b32_e32 v44, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s55, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s92, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v6
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s53, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s76, 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v5
+; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v9
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v12, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: s_lshl_b32 s4, s51, 16
+; SI-NEXT: v_lshr_b64 v[32:33], v[26:27], 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s60, 16
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v11
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[18:19], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s59, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[14:15], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[10:11], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v28
-; SI-NEXT: v_mov_b32_e32 v15, v16
-; SI-NEXT: v_mov_b32_e32 v16, v17
-; SI-NEXT: v_mov_b32_e32 v19, v36
-; SI-NEXT: v_mov_b32_e32 v17, v39
-; SI-NEXT: v_mov_b32_e32 v18, v51
-; SI-NEXT: v_mov_b32_e32 v14, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[27:28], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v27, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s49, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[34:35], 16
-; SI-NEXT: v_mov_b32_e32 v1, v32
-; SI-NEXT: v_lshr_b64 v[32:33], v[31:32], 16
-; SI-NEXT: v_lshr_b64 v[33:34], v[6:7], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v35, v15
-; SI-NEXT: v_mov_b32_e32 v28, v48
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v31, v0
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v22, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v1
+; SI-NEXT: s_lshl_b32 s4, s58, 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[27:28], 16
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s39, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s37, 16
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v2
+; SI-NEXT: v_lshr_b64 v[43:44], v[26:27], 16
+; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v13
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_mov_b32_e32 v53, v1
+; SI-NEXT: v_mov_b32_e32 v52, v0
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s35, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[36:37], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v37, v8
-; SI-NEXT: v_mov_b32_e32 v36, v24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshr_b64 v[1:2], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
+; SI-NEXT: v_mov_b32_e32 v51, v1
+; SI-NEXT: v_mov_b32_e32 v50, v0
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v48, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s64, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s89, 16
+; SI-NEXT: v_add_f32_e32 v25, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s54, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s52, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v60, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s50, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v14
+; SI-NEXT: v_lshr_b64 v[3:4], v[60:61], 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v16
+; SI-NEXT: s_lshl_b32 s4, s79, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v17
+; SI-NEXT: v_lshr_b64 v[17:18], v[4:5], 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v12
+; SI-NEXT: v_add_f32_e32 v18, s4, v23
+; SI-NEXT: v_lshr_b64 v[11:12], v[29:30], 16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v18
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v19
+; SI-NEXT: v_lshr_b64 v[19:20], v[6:7], 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s45, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v20
+; SI-NEXT: s_lshl_b32 s4, s74, 16
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v21
+; SI-NEXT: v_lshr_b64 v[21:22], v[12:13], 16
+; SI-NEXT: v_add_f32_e32 v59, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s48, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v10
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v24
+; SI-NEXT: v_lshr_b64 v[23:24], v[59:60], 16
+; SI-NEXT: v_lshr_b64 v[40:41], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[34:35], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[45:46], v[25:26], 16
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v15
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
+; SI-NEXT: v_mov_b32_e32 v15, v3
+; SI-NEXT: v_mov_b32_e32 v14, v2
+; SI-NEXT: v_lshr_b64 v[36:37], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[18:19], 16
+; SI-NEXT: v_mov_b32_e32 v12, v11
+; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
+; SI-NEXT: v_lshr_b64 v[37:38], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[14:15], 16
+; SI-NEXT: v_mov_b32_e32 v9, v15
+; SI-NEXT: v_mov_b32_e32 v11, v45
+; SI-NEXT: v_lshr_b64 v[14:15], v[44:45], 16
+; SI-NEXT: v_mov_b32_e32 v59, v34
+; SI-NEXT: v_lshr_b64 v[44:45], v[33:34], 16
+; SI-NEXT: v_mov_b32_e32 v46, v40
+; SI-NEXT: v_lshr_b64 v[34:35], v[39:40], 16
+; SI-NEXT: v_mov_b32_e32 v40, v50
+; SI-NEXT: v_mov_b32_e32 v41, v51
+; SI-NEXT: v_lshr_b64 v[56:57], v[40:41], 16
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v33, v32
+; SI-NEXT: v_lshr_b64 v[24:25], v[31:32], 16
+; SI-NEXT: v_mov_b32_e32 v35, v28
+; SI-NEXT: v_lshr_b64 v[2:3], v[16:17], 16
+; SI-NEXT: v_mov_b32_e32 v15, v52
+; SI-NEXT: v_mov_b32_e32 v16, v53
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v45, v43
+; SI-NEXT: v_lshr_b64 v[47:48], v[42:43], 16
+; SI-NEXT: v_lshr_b64 v[57:58], v[15:16], 16
+; SI-NEXT: v_mov_b32_e32 v55, v5
+; SI-NEXT: v_mov_b32_e32 v18, v14
+; SI-NEXT: v_mov_b32_e32 v14, v30
+; SI-NEXT: v_mov_b32_e32 v25, v27
+; SI-NEXT: v_mov_b32_e32 v30, v47
+; SI-NEXT: v_mov_b32_e32 v5, v13
+; SI-NEXT: v_mov_b32_e32 v48, v10
+; SI-NEXT: v_mov_b32_e32 v27, v2
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_mov_b32_e32 v31, v38
+; SI-NEXT: v_mov_b32_e32 v32, v39
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshr_b64 v[50:51], v[51:52], 16
+; SI-NEXT: v_mov_b32_e32 v51, v32
+; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_mov_b32_e32 v53, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v28, v38
+; SI-NEXT: v_mov_b32_e32 v29, v39
+; SI-NEXT: v_mov_b32_e32 v39, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v32, v29
+; SI-NEXT: v_lshr_b64 v[42:43], v[28:29], 16
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v29, v24
+; SI-NEXT: v_mov_b32_e32 v40, v42
+; SI-NEXT: v_mov_b32_e32 v43, v57
+; SI-NEXT: v_mov_b32_e32 v57, v8
+; SI-NEXT: v_mov_b32_e32 v24, v56
+; SI-NEXT: v_mov_b32_e32 v38, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[15:16], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v26, v29
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[59:60], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[57:58], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[55:56], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[46:47], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[41:42], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[39:40], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[8:9], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[20:21], 16
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v16, v41
-; SI-NEXT: v_lshr_b64 v[39:40], v[61:62], 16
-; SI-NEXT: v_mov_b32_e32 v8, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v11, v10
-; SI-NEXT: v_mov_b32_e32 v10, v13
-; SI-NEXT: s_branch .LBB101_3
-; SI-NEXT: .LBB101_2:
-; SI-NEXT: v_mov_b32_e32 v19, v13
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: .LBB101_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_cbranch_scc1 .LBB101_5
-; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v13, v19
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v59
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v61
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v22, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v20, v0
+; SI-NEXT: s_branch .LBB101_6
+; SI-NEXT: .LBB101_5:
+; SI-NEXT: v_mov_b32_e32 v1, s60
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v18
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, s76
+; SI-NEXT: v_readlane_b32 s4, v62, 39
+; SI-NEXT: v_mov_b32_e32 v60, s48
+; SI-NEXT: v_mov_b32_e32 v5, s45
+; SI-NEXT: v_mov_b32_e32 v7, s73
+; SI-NEXT: v_mov_b32_e32 v55, s75
+; SI-NEXT: v_mov_b32_e32 v61, s50
+; SI-NEXT: v_mov_b32_e32 v14, s52
+; SI-NEXT: v_mov_b32_e32 v26, s54
+; SI-NEXT: v_mov_b32_e32 v49, s64
+; SI-NEXT: v_mov_b32_e32 v39, s25
+; SI-NEXT: v_mov_b32_e32 v53, s35
+; SI-NEXT: v_mov_b32_e32 v25, s39
+; SI-NEXT: v_mov_b32_e32 v35, s49
+; SI-NEXT: v_mov_b32_e32 v18, s44
+; SI-NEXT: v_mov_b32_e32 v31, s46
+; SI-NEXT: v_mov_b32_e32 v40, s56
+; SI-NEXT: v_mov_b32_e32 v12, s41
+; SI-NEXT: v_mov_b32_e32 v11, s95
+; SI-NEXT: v_mov_b32_e32 v59, s85
+; SI-NEXT: v_mov_b32_e32 v46, s91
+; SI-NEXT: v_mov_b32_e32 v41, s87
+; SI-NEXT: v_mov_b32_e32 v45, s97
+; SI-NEXT: v_mov_b32_e32 v16, s11
+; SI-NEXT: v_mov_b32_e32 v52, s15
+; SI-NEXT: v_mov_b32_e32 v33, s7
+; SI-NEXT: v_mov_b32_e32 v51, s43
+; SI-NEXT: v_mov_b32_e32 v32, s13
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v22
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v38
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v11
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v35
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v49
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v37
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v61, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v59, 0x40c00000, v0
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v50, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v2
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v44, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v1
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v54, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v42, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v34
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v1, s92
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v56
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, s30
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v58
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v57, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v57
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v58, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v7
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v14
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v28
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v14
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v35
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v29
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v18
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v21
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v16
-; SI-NEXT: v_lshr_b64 v[16:17], v[50:51], 16
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v57
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v22
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v30
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v23
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v23
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v30
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v48
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v35
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v37
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v36
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v47
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v37
-; SI-NEXT: v_lshr_b64 v[46:47], v[10:11], 16
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v33
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, s14
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v2
-; SI-NEXT: v_mov_b32_e32 v2, v1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 40
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v48
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: v_mov_b32_e32 v10, v60
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[8:9], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v48
-; SI-NEXT: v_lshr_b64 v[48:49], v[59:60], 16
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v37
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v21
-; SI-NEXT: v_lshr_b64 v[36:37], v[26:27], 16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v22
-; SI-NEXT: v_lshr_b64 v[22:23], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[38:39], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v35
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[60:61], v[61:62], 16
-; SI-NEXT: v_mov_b32_e32 v29, v10
-; SI-NEXT: v_mov_b32_e32 v28, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v34
-; SI-NEXT: v_lshr_b64 v[10:11], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v39, v46
-; SI-NEXT: v_mov_b32_e32 v43, v22
-; SI-NEXT: v_lshr_b64 v[32:33], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v1, v62
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v25
-; SI-NEXT: v_lshr_b64 v[25:26], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[52:53], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v58
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[52:53], v[54:55], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[9:10], 16
-; SI-NEXT: v_mov_b32_e32 v44, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_mov_b32_e32 v30, v32
-; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
-; SI-NEXT: v_lshr_b64 v[32:33], v[47:48], 16
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v51
-; SI-NEXT: v_lshr_b64 v[50:51], v[40:41], 16
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v56
-; SI-NEXT: v_lshr_b64 v[53:54], v[51:52], 16
-; SI-NEXT: v_mov_b32_e32 v54, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v0, v2
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[17:18], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[12:13], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[21:22], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[24:25], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 41
+; SI-NEXT: v_mov_b32_e32 v38, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 42
+; SI-NEXT: v_mov_b32_e32 v28, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 43
+; SI-NEXT: v_mov_b32_e32 v22, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 44
+; SI-NEXT: v_mov_b32_e32 v20, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 45
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, v36
-; SI-NEXT: v_mov_b32_e32 v12, v50
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 46
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[35:36], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, v29
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[49:50], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[15:16], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v50, v16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v3, v1
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 47
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[7:8], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 48
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[59:60], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v48
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: .LBB101_5: ; %end
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 49
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 50
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 51
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 52
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 53
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 3
+; SI-NEXT: v_readlane_b32 s5, v62, 4
+; SI-NEXT: v_mov_b32_e32 v36, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 7
+; SI-NEXT: v_readlane_b32 s5, v62, 8
+; SI-NEXT: v_mov_b32_e32 v37, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 11
+; SI-NEXT: v_readlane_b32 s5, v62, 12
+; SI-NEXT: v_mov_b32_e32 v57, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 15
+; SI-NEXT: v_readlane_b32 s5, v62, 16
+; SI-NEXT: v_mov_b32_e32 v27, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 19
+; SI-NEXT: v_readlane_b32 s5, v62, 20
+; SI-NEXT: v_mov_b32_e32 v54, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 23
+; SI-NEXT: v_readlane_b32 s5, v62, 24
+; SI-NEXT: v_mov_b32_e32 v48, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 25
+; SI-NEXT: v_readlane_b32 s5, v62, 26
+; SI-NEXT: v_mov_b32_e32 v44, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 27
+; SI-NEXT: v_readlane_b32 s5, v62, 28
+; SI-NEXT: v_mov_b32_e32 v34, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 29
+; SI-NEXT: v_readlane_b32 s5, v62, 30
+; SI-NEXT: v_mov_b32_e32 v24, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 31
+; SI-NEXT: v_readlane_b32 s5, v62, 32
+; SI-NEXT: v_mov_b32_e32 v30, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 33
+; SI-NEXT: v_readlane_b32 s5, v62, 34
+; SI-NEXT: v_mov_b32_e32 v43, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 35
+; SI-NEXT: v_readlane_b32 s5, v62, 36
+; SI-NEXT: v_mov_b32_e32 v50, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 37
+; SI-NEXT: v_readlane_b32 s5, v62, 38
+; SI-NEXT: v_mov_b32_e32 v29, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 5
+; SI-NEXT: v_readlane_b32 s5, v62, 6
+; SI-NEXT: v_mov_b32_e32 v23, s5
+; SI-NEXT: v_readlane_b32 s4, v62, 9
+; SI-NEXT: v_readlane_b32 s5, v62, 10
+; SI-NEXT: v_mov_b32_e32 v21, s5
+; SI-NEXT: v_readlane_b32 s4, v62, 13
+; SI-NEXT: v_readlane_b32 s5, v62, 14
+; SI-NEXT: v_mov_b32_e32 v19, s5
+; SI-NEXT: v_readlane_b32 s4, v62, 17
+; SI-NEXT: v_readlane_b32 s5, v62, 18
+; SI-NEXT: v_mov_b32_e32 v17, s5
+; SI-NEXT: v_readlane_b32 s4, v62, 21
+; SI-NEXT: v_readlane_b32 s5, v62, 22
+; SI-NEXT: v_mov_b32_e32 v9, s5
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: .LBB101_6: ; %end
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v26
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v10
-; SI-NEXT: v_and_b32_e32 v25, 0xffff, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v52
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v18
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v12
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v43
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v44
-; SI-NEXT: v_and_b32_e32 v21, 0xffff, v50
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v54
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s30, v63, 3
-; SI-NEXT: v_readlane_b32 s31, v63, 4
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v36
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v44
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v59
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v46
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v41
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v45
+; SI-NEXT: v_readlane_b32 s30, v63, 34
+; SI-NEXT: v_readlane_b32 s31, v63, 35
+; SI-NEXT: v_readlane_b32 s99, v63, 33
+; SI-NEXT: v_readlane_b32 s98, v63, 32
+; SI-NEXT: v_readlane_b32 s97, v63, 31
+; SI-NEXT: v_readlane_b32 s96, v63, 30
+; SI-NEXT: v_readlane_b32 s87, v63, 29
+; SI-NEXT: v_readlane_b32 s86, v63, 28
+; SI-NEXT: v_readlane_b32 s85, v63, 27
+; SI-NEXT: v_readlane_b32 s84, v63, 26
+; SI-NEXT: v_readlane_b32 s83, v63, 25
+; SI-NEXT: v_readlane_b32 s82, v63, 24
+; SI-NEXT: v_readlane_b32 s81, v63, 23
+; SI-NEXT: v_readlane_b32 s80, v63, 22
+; SI-NEXT: v_readlane_b32 s71, v63, 21
+; SI-NEXT: v_readlane_b32 s70, v63, 20
+; SI-NEXT: v_readlane_b32 s69, v63, 19
+; SI-NEXT: v_readlane_b32 s68, v63, 18
+; SI-NEXT: v_readlane_b32 s67, v63, 17
+; SI-NEXT: v_readlane_b32 s66, v63, 16
+; SI-NEXT: v_readlane_b32 s65, v63, 15
+; SI-NEXT: v_readlane_b32 s64, v63, 14
+; SI-NEXT: v_readlane_b32 s55, v63, 13
+; SI-NEXT: v_readlane_b32 s54, v63, 12
+; SI-NEXT: v_readlane_b32 s53, v63, 11
+; SI-NEXT: v_readlane_b32 s52, v63, 10
+; SI-NEXT: v_readlane_b32 s51, v63, 9
+; SI-NEXT: v_readlane_b32 s50, v63, 8
+; SI-NEXT: v_readlane_b32 s49, v63, 7
+; SI-NEXT: v_readlane_b32 s48, v63, 6
+; SI-NEXT: v_readlane_b32 s39, v63, 5
+; SI-NEXT: v_readlane_b32 s38, v63, 4
+; SI-NEXT: v_readlane_b32 s37, v63, 3
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v14
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v14
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v60
; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v53
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v37
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v57
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v27
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_or_b32_e32 v6, v6, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v55
+; SI-NEXT: v_or_b32_e32 v7, v8, v7
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v54
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v61
; SI-NEXT: v_or_b32_e32 v9, v10, v9
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_or_b32_e32 v10, v10, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v13
-; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_or_b32_e32 v12, v12, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v28
-; SI-NEXT: v_or_b32_e32 v13, v16, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: v_or_b32_e32 v14, v14, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_or_b32_e32 v16, v16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; SI-NEXT: v_or_b32_e32 v17, v19, v17
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v18, v18, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v20
-; SI-NEXT: v_or_b32_e32 v19, v21, v19
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_or_b32_e32 v20, v20, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v22
-; SI-NEXT: v_or_b32_e32 v21, v23, v21
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_or_b32_e32 v22, v22, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; SI-NEXT: v_or_b32_e32 v23, v25, v23
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v48
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v14
+; SI-NEXT: v_or_b32_e32 v11, v12, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v36
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; SI-NEXT: v_or_b32_e32 v12, v12, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v26
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v34
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v36
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v49
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v36
+; SI-NEXT: v_or_b32_e32 v16, v16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v39
+; SI-NEXT: v_or_b32_e32 v17, v17, v18
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v24
+; SI-NEXT: v_or_b32_e32 v18, v18, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v53
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v30
+; SI-NEXT: v_or_b32_e32 v20, v20, v22
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v43
+; SI-NEXT: v_or_b32_e32 v22, v22, v24
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v35
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v38
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v50
; SI-NEXT: v_or_b32_e32 v24, v24, v25
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v26
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v52
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; SI-NEXT: v_or_b32_e32 v25, v27, v25
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v39
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v29
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
; SI-NEXT: v_or_b32_e32 v26, v26, v27
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff, v28
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v33
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; SI-NEXT: v_or_b32_e32 v27, v29, v27
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
; SI-NEXT: v_or_b32_e32 v28, v28, v29
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v51
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v31
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v30
-; SI-NEXT: v_or_b32_e32 v29, v31, v29
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v33
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v40
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
; SI-NEXT: v_or_b32_e32 v30, v30, v31
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v31, 0xffff, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -216167,1048 +211579,950 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v3
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v2
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v60
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB102_2
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v58
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v59
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v60
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v61
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v22
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v20
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v16
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v14
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v32
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v12
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v10
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v8
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v6
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v34
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v2
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v35
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v0
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v36
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v25
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v1
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; kill: killed $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; kill: killed $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; kill: killed $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v36
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v42
+; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB102_2
+; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v54
+; SI-NEXT: v_or_b32_e32 v56, v1, v26
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v53
+; SI-NEXT: v_or_b32_e32 v59, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v52
+; SI-NEXT: v_or_b32_e32 v63, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v51
+; SI-NEXT: v_or_b32_e32 v62, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v50
+; SI-NEXT: v_or_b32_e32 v60, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v49
+; SI-NEXT: v_or_b32_e32 v58, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v48
+; SI-NEXT: v_or_b32_e32 v57, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v39
+; SI-NEXT: v_or_b32_e32 v47, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v38
+; SI-NEXT: v_or_b32_e32 v46, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v37
+; SI-NEXT: v_or_b32_e32 v45, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v35
+; SI-NEXT: v_or_b32_e32 v44, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v34
+; SI-NEXT: v_or_b32_e32 v43, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v33
+; SI-NEXT: v_or_b32_e32 v9, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v32
+; SI-NEXT: v_or_b32_e32 v7, v1, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v29
+; SI-NEXT: v_mov_b32_e32 v26, v9
+; SI-NEXT: v_mov_b32_e32 v28, v7
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v5, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v42
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v36
+; SI-NEXT: v_or_b32_e32 v3, v1, v3
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v56, v1, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v38
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v59, v0, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v39
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v63, v0, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v48
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v49
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v62, v0, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v50
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v60, v0, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v51
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v52
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v53
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v17
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v58, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v54
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v12
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v57, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v55
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v14
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v19
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v47, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v46, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v18
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v21
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v45, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v20
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: v_alignbit_b32 v0, v44, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v22
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v23
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v0, v43, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: v_alignbit_b32 v55, v9, v0, 16
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; kill: killed $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: v_mov_b32_e32 v31, v5
+; SI-NEXT: v_mov_b32_e32 v40, v3
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; kill: killed $vgpr24
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; kill: killed $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v61, v7, v0, 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v41, v5, v0, 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v30, v3, v0, 16
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: .LBB102_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_4
; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v26, v36
+; SI-NEXT: v_mov_b32_e32 v31, v24
+; SI-NEXT: v_cvt_f32_f16_e32 v24, v42
+; SI-NEXT: v_cvt_f32_f16_e32 v25, v25
+; SI-NEXT: v_add_f32_e32 v26, 0x38000000, v26
+; SI-NEXT: v_cvt_f16_f32_e32 v36, v26
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT: v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT: v_cvt_f32_f16_e32 v23, v23
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v36
+; SI-NEXT: v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT: v_or_b32_e32 v30, v24, v26
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v26, v29
+; SI-NEXT: v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT: v_cvt_f16_f32_e32 v23, v23
+; SI-NEXT: v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT: v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT: v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT: v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v15
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_mov_b32_e32 v40, v30
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v24, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT: v_cvt_f16_f32_e32 v28, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v26
+; SI-NEXT: v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v29, v24, v26
+; SI-NEXT: v_cvt_f32_f16_e32 v24, v32
+; SI-NEXT: v_cvt_f32_f16_e32 v26, v27
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT: v_cvt_f16_f32_e32 v32, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v26
+; SI-NEXT: v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v32
+; SI-NEXT: v_or_b32_e32 v27, v24, v26
+; SI-NEXT: v_cvt_f32_f16_e32 v24, v33
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v28, v27
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT: v_cvt_f16_f32_e32 v33, v24
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v25
+; SI-NEXT: v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v33
+; SI-NEXT: v_or_b32_e32 v25, v24, v25
+; SI-NEXT: v_cvt_f32_f16_e32 v24, v34
+; SI-NEXT: v_mov_b32_e32 v26, v25
+; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT: v_cvt_f16_f32_e32 v34, v24
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v34
+; SI-NEXT: v_or_b32_e32 v43, v23, v24
+; SI-NEXT: v_cvt_f32_f16_e32 v23, v35
+; SI-NEXT: v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT: v_cvt_f16_f32_e32 v35, v23
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v35
+; SI-NEXT: v_or_b32_e32 v44, v21, v23
+; SI-NEXT: v_cvt_f32_f16_e32 v21, v37
+; SI-NEXT: v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT: v_cvt_f16_f32_e32 v37, v21
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v37
+; SI-NEXT: v_or_b32_e32 v45, v19, v21
+; SI-NEXT: v_cvt_f32_f16_e32 v19, v38
+; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT: v_cvt_f16_f32_e32 v38, v19
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v38
+; SI-NEXT: v_or_b32_e32 v46, v17, v19
+; SI-NEXT: v_cvt_f32_f16_e32 v17, v39
+; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT: v_cvt_f16_f32_e32 v39, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v39
+; SI-NEXT: v_or_b32_e32 v47, v15, v17
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v48
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v48, v15
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v48
+; SI-NEXT: v_or_b32_e32 v57, v13, v15
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v49
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v49, v13
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v49
+; SI-NEXT: v_or_b32_e32 v58, v11, v13
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v50
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v31
+; SI-NEXT: v_mov_b32_e32 v31, v29
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v50, v11
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v50
+; SI-NEXT: v_or_b32_e32 v60, v9, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v51
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v18
+; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT: v_cvt_f16_f32_e32 v51, v9
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v51
+; SI-NEXT: v_or_b32_e32 v62, v7, v9
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v52
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v52, v7
+; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v52
+; SI-NEXT: v_or_b32_e32 v63, v5, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v53
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v53, v5
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v53
+; SI-NEXT: v_or_b32_e32 v59, v3, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v54
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v8
+; SI-NEXT: v_cvt_f32_f16_e32 v8, v12
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v54, v3
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v54
+; SI-NEXT: v_or_b32_e32 v56, v1, v3
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
+; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v20
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_alignbit_b32 v1, v56, v1, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v31, v61
-; SI-NEXT: v_cvt_f32_f16_e32 v33, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v59
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v58
-; SI-NEXT: v_add_f32_e32 v31, 0x38000000, v31
-; SI-NEXT: v_add_f32_e32 v59, 0x38000000, v33
-; SI-NEXT: v_cvt_f16_f32_e32 v33, v31
-; SI-NEXT: v_cvt_f32_f16_e32 v31, v30
-; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v29
-; SI-NEXT: v_add_f32_e32 v29, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v57
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v59, v3, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v57, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v28
-; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v56
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_add_f32_e32 v56, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v27
-; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v47
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_add_f32_e32 v47, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v26
-; SI-NEXT: v_add_f32_e32 v26, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v46
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_add_f32_e32 v46, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v25
-; SI-NEXT: v_add_f32_e32 v25, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v45
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v45, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v24
-; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v44
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_add_f32_e32 v44, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v43, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v42
-; SI-NEXT: v_cvt_f32_f16_e32 v32, v32
-; SI-NEXT: v_add_f32_e32 v42, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v41
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_add_f32_e32 v41, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v40
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT: v_add_f32_e32 v40, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v19
-; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v55
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v55, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v54
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_add_f32_e32 v54, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v53
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_add_f32_e32 v53, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v52
+; SI-NEXT: v_or_b32_e32 v2, v2, v0
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v63, v0, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v5, v5, v2
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v0, v62, v2, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT: v_add_f32_e32 v52, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v51
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v51, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v50
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_add_f32_e32 v50, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v49
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f32_e32 v49, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v58, v48
-; SI-NEXT: v_cvt_f32_f16_e32 v34, v34
-; SI-NEXT: v_add_f32_e32 v48, 0x38000000, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v58
-; SI-NEXT: v_add_f32_e32 v32, 0x38000000, v32
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_add_f32_e32 v58, 0x38000000, v60
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v32, v32
-; SI-NEXT: v_cvt_f32_f16_e32 v35, v35
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v36, v36
-; SI-NEXT: v_add_f32_e32 v34, 0x38000000, v34
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v6, v6, v4
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v10
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v0, v60, v4, 16
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT: v_cvt_f16_f32_e32 v34, v34
-; SI-NEXT: v_cvt_f32_f16_e32 v37, v37
-; SI-NEXT: v_add_f32_e32 v35, 0x38000000, v35
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v0, v58, v5, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v6, v6, v5
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v8, v8, v6
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v0, v57, v6, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT: v_cvt_f16_f32_e32 v35, v35
-; SI-NEXT: v_cvt_f32_f16_e32 v38, v38
-; SI-NEXT: v_add_f32_e32 v36, 0x38000000, v36
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v10, v10
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v36, v36
-; SI-NEXT: v_cvt_f32_f16_e32 v39, v39
-; SI-NEXT: v_add_f32_e32 v37, 0x38000000, v37
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_or_b32_e32 v9, v9, v7
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT: v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT: v_add_f32_e32 v38, 0x38000000, v38
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
-; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT: v_cvt_f16_f32_e32 v38, v38
-; SI-NEXT: v_add_f32_e32 v39, 0x38000000, v39
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT: v_cvt_f16_f32_e32 v39, v39
-; SI-NEXT: v_cvt_f16_f32_e32 v58, v58
-; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT: v_cvt_f16_f32_e32 v48, v48
-; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT: v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT: v_cvt_f16_f32_e32 v50, v50
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v51, v51
+; SI-NEXT: v_alignbit_b32 v0, v47, v7, 16
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT: v_cvt_f16_f32_e32 v52, v52
-; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v53, v53
-; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT: v_cvt_f16_f32_e32 v54, v54
-; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT: v_cvt_f16_f32_e32 v55, v55
-; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_cvt_f32_f16_e32 v61, v61
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_cvt_f32_f16_e32 v62, v62
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_cvt_f32_f16_e32 v63, v63
-; SI-NEXT: v_cvt_f16_f32_e32 v40, v40
-; SI-NEXT: v_add_f32_e32 v61, 0x38000000, v61
-; SI-NEXT: v_add_f32_e32 v62, 0x38000000, v62
-; SI-NEXT: v_add_f32_e32 v63, 0x38000000, v63
-; SI-NEXT: v_cvt_f16_f32_e32 v63, v63
-; SI-NEXT: v_cvt_f16_f32_e32 v62, v62
-; SI-NEXT: v_cvt_f16_f32_e32 v61, v61
-; SI-NEXT: v_cvt_f16_f32_e32 v20, v20
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v63
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v62
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v61
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v32
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_cvt_f32_f16_e32 v60, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f32_e32 v60, 0x38000000, v60
-; SI-NEXT: v_cvt_f16_f32_e32 v60, v60
-; SI-NEXT: v_cvt_f16_f32_e32 v41, v41
-; SI-NEXT: v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT: v_cvt_f16_f32_e32 v42, v42
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v34
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v35
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v36
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v38
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v39
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v9, v9, v8
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v48
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v49
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v11, v11, v9
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v13
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v50
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v51
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v52
+; SI-NEXT: v_alignbit_b32 v0, v46, v8, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v53
+; SI-NEXT: v_alignbit_b32 v0, v45, v9, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_or_b32_e32 v12, v12, v10
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v17
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v54
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v22
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_alignbit_b32 v0, v44, v10, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v55
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v19
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v40
-; SI-NEXT: v_cvt_f16_f32_e32 v22, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT: v_cvt_f16_f32_e32 v43, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v41
-; SI-NEXT: v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v21
-; SI-NEXT: v_cvt_f16_f32_e32 v44, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v42
-; SI-NEXT: v_cvt_f16_f32_e32 v24, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_cvt_f16_f32_e32 v45, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v43
-; SI-NEXT: v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v23
-; SI-NEXT: v_cvt_f16_f32_e32 v46, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v44
-; SI-NEXT: v_cvt_f16_f32_e32 v26, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT: v_cvt_f16_f32_e32 v47, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v45
-; SI-NEXT: v_cvt_f16_f32_e32 v27, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v25
-; SI-NEXT: v_cvt_f16_f32_e32 v56, v56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v46
-; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v26
-; SI-NEXT: v_add_f32_e32 v31, 0x38000000, v31
-; SI-NEXT: v_cvt_f16_f32_e32 v57, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v47
-; SI-NEXT: v_cvt_f16_f32_e32 v59, v59
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v30
-; SI-NEXT: v_cvt_f16_f32_e32 v31, v31
-; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v0, v43, v11, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v12, v12, v11
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v55, v25, v12, 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_or_b32_e32 v15, v15, v13
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT: v_alignbit_b32 v61, v27, v13, 16
+; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v12
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v29
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_or_b32_e32 v15, v15, v14
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v59
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; SI-NEXT: v_or_b32_e32 v16, v17, v15
+; SI-NEXT: v_alignbit_b32 v41, v29, v14, 16
+; SI-NEXT: v_alignbit_b32 v30, v30, v15, 16
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: .LBB102_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v62
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v54
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v53
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v52
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v51
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v50
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v49
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v48
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v38
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v37
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v35
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v34
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v55
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v61
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v41
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v56
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v63
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v62
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v8, v8, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v60
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v58
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v57
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v47
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v16, v16, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v46
+; SI-NEXT: v_or_b32_e32 v17, v17, v18
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v18, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v45
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v44
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v43
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v33
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v32
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v36
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v31
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_or_b32_e32 v29, v29, v31
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v63
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_or_b32_e32 v30, v31, v30
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v40
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@@ -217225,17 +212539,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v31
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64f16_to_v64bf16:
@@ -217477,8 +212782,8 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -217530,791 +212835,886 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_writelane_b32 v63, s99, 33
; SI-NEXT: v_writelane_b32 v63, s30, 34
; SI-NEXT: v_writelane_b32 v63, s31, 35
-; SI-NEXT: s_lshr_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s4, s17, 16
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 s7, s17, 16
-; SI-NEXT: v_writelane_b32 v62, s6, 0
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: v_writelane_b32 v62, s7, 1
-; SI-NEXT: s_lshr_b32 s9, s19, 16
-; SI-NEXT: v_writelane_b32 v62, s8, 2
-; SI-NEXT: s_lshr_b32 s10, s20, 16
-; SI-NEXT: v_writelane_b32 v62, s9, 3
-; SI-NEXT: s_lshr_b32 s11, s21, 16
-; SI-NEXT: v_writelane_b32 v62, s10, 4
-; SI-NEXT: s_lshr_b32 s12, s22, 16
-; SI-NEXT: v_readfirstlane_b32 s49, v17
-; SI-NEXT: v_readfirstlane_b32 s37, v16
-; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v14
-; SI-NEXT: v_readfirstlane_b32 s94, v13
-; SI-NEXT: v_readfirstlane_b32 s92, v12
-; SI-NEXT: v_readfirstlane_b32 s90, v11
-; SI-NEXT: v_readfirstlane_b32 s88, v10
-; SI-NEXT: v_readfirstlane_b32 s78, v9
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: v_readfirstlane_b32 s74, v7
-; SI-NEXT: v_readfirstlane_b32 s72, v6
-; SI-NEXT: v_readfirstlane_b32 s62, v5
-; SI-NEXT: v_readfirstlane_b32 s60, v4
-; SI-NEXT: v_readfirstlane_b32 s58, v3
-; SI-NEXT: v_readfirstlane_b32 s56, v2
-; SI-NEXT: v_readfirstlane_b32 s46, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: v_writelane_b32 v62, s11, 5
-; SI-NEXT: s_lshr_b32 s43, s29, 16
-; SI-NEXT: s_lshr_b32 s42, s28, 16
-; SI-NEXT: s_lshr_b32 s41, s27, 16
-; SI-NEXT: s_lshr_b32 s40, s26, 16
-; SI-NEXT: s_lshr_b32 s15, s25, 16
-; SI-NEXT: s_lshr_b32 s14, s24, 16
-; SI-NEXT: s_lshr_b32 s13, s23, 16
-; SI-NEXT: s_lshr_b32 s52, s49, 16
-; SI-NEXT: s_lshr_b32 s48, s37, 16
-; SI-NEXT: s_lshr_b32 s35, s4, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
-; SI-NEXT: s_lshr_b32 s95, s94, 16
-; SI-NEXT: s_lshr_b32 s93, s92, 16
-; SI-NEXT: s_lshr_b32 s91, s90, 16
-; SI-NEXT: s_lshr_b32 s89, s88, 16
-; SI-NEXT: s_lshr_b32 s79, s78, 16
-; SI-NEXT: s_lshr_b32 s77, s76, 16
-; SI-NEXT: s_lshr_b32 s75, s74, 16
-; SI-NEXT: s_lshr_b32 s73, s72, 16
-; SI-NEXT: s_lshr_b32 s63, s62, 16
-; SI-NEXT: s_lshr_b32 s61, s60, 16
-; SI-NEXT: s_lshr_b32 s59, s58, 16
-; SI-NEXT: s_lshr_b32 s57, s56, 16
-; SI-NEXT: s_lshr_b32 s47, s46, 16
-; SI-NEXT: s_lshr_b32 s45, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v18
-; SI-NEXT: v_writelane_b32 v62, s12, 6
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_writelane_b32 v62, s13, 7
+; SI-NEXT: s_mov_b32 s91, s26
+; SI-NEXT: v_writelane_b32 v62, s4, 0
+; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: v_writelane_b32 v62, s4, 1
+; SI-NEXT: s_lshr_b32 s51, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v16
+; SI-NEXT: v_writelane_b32 v62, s4, 2
+; SI-NEXT: s_lshr_b32 s53, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v14
+; SI-NEXT: v_readfirstlane_b32 s79, v15
+; SI-NEXT: v_writelane_b32 v62, s4, 3
+; SI-NEXT: s_lshr_b32 s54, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s49, v13
+; SI-NEXT: v_readfirstlane_b32 s4, v12
+; SI-NEXT: v_readfirstlane_b32 s95, v11
+; SI-NEXT: v_readfirstlane_b32 s55, v10
+; SI-NEXT: v_readfirstlane_b32 s88, v9
+; SI-NEXT: v_readfirstlane_b32 s77, v8
+; SI-NEXT: v_readfirstlane_b32 s62, v7
+; SI-NEXT: v_readfirstlane_b32 s87, v6
+; SI-NEXT: v_readfirstlane_b32 s84, v5
+; SI-NEXT: v_readfirstlane_b32 s97, v4
+; SI-NEXT: v_readfirstlane_b32 s82, v3
+; SI-NEXT: v_readfirstlane_b32 s5, v2
+; SI-NEXT: v_readfirstlane_b32 s80, v1
+; SI-NEXT: v_readfirstlane_b32 s42, v0
+; SI-NEXT: s_mov_b32 s35, s22
+; SI-NEXT: s_mov_b32 s26, s21
+; SI-NEXT: s_lshr_b32 s71, s29, 16
+; SI-NEXT: s_lshr_b32 s41, s28, 16
+; SI-NEXT: s_lshr_b32 s70, s27, 16
+; SI-NEXT: s_lshr_b32 s14, s91, 16
+; SI-NEXT: s_lshr_b32 s69, s25, 16
+; SI-NEXT: s_lshr_b32 s12, s24, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 16
+; SI-NEXT: s_lshr_b32 s10, s22, 16
+; SI-NEXT: s_lshr_b32 s52, s21, 16
+; SI-NEXT: s_lshr_b32 s22, s20, 16
+; SI-NEXT: s_lshr_b32 s44, s19, 16
+; SI-NEXT: s_lshr_b32 s6, s18, 16
+; SI-NEXT: s_lshr_b32 s21, s16, 16
+; SI-NEXT: s_lshr_b32 s31, s79, 16
+; SI-NEXT: s_lshr_b32 s38, s49, 16
+; SI-NEXT: v_writelane_b32 v62, s4, 4
+; SI-NEXT: s_lshr_b32 s48, s4, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s95, 16
+; SI-NEXT: s_lshr_b32 s34, s55, 16
+; SI-NEXT: s_lshr_b32 s86, s88, 16
+; SI-NEXT: s_lshr_b32 s96, s77, 16
+; SI-NEXT: s_lshr_b32 s85, s62, 16
+; SI-NEXT: s_lshr_b32 s98, s87, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s65, s97, 16
+; SI-NEXT: s_lshr_b32 s81, s82, 16
+; SI-NEXT: s_lshr_b32 s59, s5, 16
+; SI-NEXT: s_lshr_b32 s46, s80, 16
+; SI-NEXT: s_lshr_b32 s99, s42, 16
+; SI-NEXT: v_readfirstlane_b32 s7, v18
+; SI-NEXT: s_mov_b32 s39, s29
+; SI-NEXT: s_mov_b32 s60, s19
+; SI-NEXT: s_mov_b32 s58, s17
+; SI-NEXT: s_cmp_lg_u32 s7, 0
+; SI-NEXT: v_writelane_b32 v62, s5, 5
; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s5, s16, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 33
-; SI-NEXT: s_lshl_b32 s5, s6, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 32
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 31
-; SI-NEXT: s_lshl_b32 s5, s7, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 30
-; SI-NEXT: s_lshl_b32 s5, s18, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 29
-; SI-NEXT: s_lshl_b32 s5, s8, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 28
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 27
-; SI-NEXT: s_lshl_b32 s5, s9, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 26
-; SI-NEXT: s_lshl_b32 s5, s20, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 25
-; SI-NEXT: s_lshl_b32 s5, s10, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 24
-; SI-NEXT: s_lshl_b32 s5, s21, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 23
-; SI-NEXT: s_lshl_b32 s5, s11, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 22
-; SI-NEXT: s_lshl_b32 s5, s22, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 21
-; SI-NEXT: s_lshl_b32 s5, s12, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 20
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 19
-; SI-NEXT: s_lshl_b32 s5, s13, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 18
-; SI-NEXT: s_lshl_b32 s5, s24, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 17
-; SI-NEXT: s_lshl_b32 s5, s14, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 16
-; SI-NEXT: s_lshl_b32 s5, s25, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 15
-; SI-NEXT: s_lshl_b32 s5, s15, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 14
-; SI-NEXT: s_lshl_b32 s5, s26, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 13
-; SI-NEXT: s_lshl_b32 s5, s40, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 12
-; SI-NEXT: s_lshl_b32 s5, s27, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 11
-; SI-NEXT: s_lshl_b32 s5, s41, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 10
-; SI-NEXT: s_lshl_b32 s5, s28, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 9
-; SI-NEXT: s_lshl_b32 s5, s42, 16
-; SI-NEXT: v_writelane_b32 v62, s5, 8
-; SI-NEXT: s_lshl_b32 s87, s29, 16
-; SI-NEXT: s_lshl_b32 s96, s43, 16
-; SI-NEXT: s_lshl_b32 s97, s44, 16
-; SI-NEXT: s_lshl_b32 s98, s45, 16
-; SI-NEXT: s_lshl_b32 s99, s46, 16
-; SI-NEXT: s_lshl_b32 s30, s47, 16
-; SI-NEXT: s_lshl_b32 s31, s56, 16
-; SI-NEXT: s_lshl_b32 s36, s57, 16
-; SI-NEXT: s_lshl_b32 s38, s58, 16
-; SI-NEXT: s_lshl_b32 s39, s59, 16
-; SI-NEXT: s_lshl_b32 s50, s60, 16
-; SI-NEXT: s_lshl_b32 s51, s61, 16
-; SI-NEXT: s_lshl_b32 s53, s62, 16
-; SI-NEXT: s_lshl_b32 s54, s63, 16
-; SI-NEXT: s_lshl_b32 s55, s72, 16
-; SI-NEXT: s_lshl_b32 s64, s73, 16
-; SI-NEXT: s_lshl_b32 s6, s74, 16
-; SI-NEXT: s_lshl_b32 s65, s75, 16
-; SI-NEXT: s_lshl_b32 s7, s76, 16
-; SI-NEXT: s_lshl_b32 s66, s77, 16
-; SI-NEXT: s_lshl_b32 s67, s78, 16
-; SI-NEXT: s_lshl_b32 s68, s79, 16
-; SI-NEXT: s_lshl_b32 s8, s88, 16
-; SI-NEXT: s_lshl_b32 s69, s89, 16
-; SI-NEXT: s_lshl_b32 s70, s90, 16
-; SI-NEXT: s_lshl_b32 s9, s91, 16
-; SI-NEXT: s_lshl_b32 s71, s92, 16
-; SI-NEXT: s_lshl_b32 s80, s93, 16
-; SI-NEXT: s_lshl_b32 s10, s94, 16
-; SI-NEXT: s_lshl_b32 s81, s95, 16
-; SI-NEXT: s_lshl_b32 s82, vcc_lo, 16
-; SI-NEXT: s_lshl_b32 s11, vcc_hi, 16
-; SI-NEXT: s_mov_b32 s34, s4
-; SI-NEXT: s_lshl_b32 s83, s4, 16
-; SI-NEXT: s_lshl_b32 s84, s35, 16
-; SI-NEXT: s_lshl_b32 s12, s37, 16
-; SI-NEXT: s_lshl_b32 s85, s48, 16
-; SI-NEXT: s_lshl_b32 s13, s49, 16
-; SI-NEXT: s_lshl_b32 s86, s52, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s7, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s36, s21, 16
+; SI-NEXT: s_or_b32 s7, s7, s36
+; SI-NEXT: v_writelane_b32 v62, s7, 35
+; SI-NEXT: s_and_b32 s7, s18, 0xffff
+; SI-NEXT: s_mov_b32 s76, s6
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s7, s7, s6
+; SI-NEXT: v_writelane_b32 v62, s7, 34
+; SI-NEXT: s_and_b32 s7, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s22, 16
+; SI-NEXT: s_or_b32 s7, s7, s40
+; SI-NEXT: v_writelane_b32 v62, s7, 33
+; SI-NEXT: s_and_b32 s7, s35, 0xffff
+; SI-NEXT: s_mov_b32 s90, s10
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s10
+; SI-NEXT: v_writelane_b32 v62, s7, 32
+; SI-NEXT: s_and_b32 s7, s24, 0xffff
+; SI-NEXT: s_mov_b32 s75, s12
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: v_writelane_b32 v62, s7, 31
+; SI-NEXT: s_and_b32 s7, s91, 0xffff
+; SI-NEXT: s_mov_b32 s94, s14
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_or_b32 s7, s7, s14
+; SI-NEXT: v_writelane_b32 v62, s7, 30
+; SI-NEXT: s_and_b32 s7, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s92, s41, 16
+; SI-NEXT: s_or_b32 s4, s7, s92
+; SI-NEXT: v_writelane_b32 v62, s4, 29
+; SI-NEXT: s_and_b32 s7, s42, 0xffff
+; SI-NEXT: s_mov_b32 s30, s42
+; SI-NEXT: s_lshl_b32 s42, s99, 16
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: s_or_b32 s61, s7, s42
+; SI-NEXT: s_and_b32 s17, s5, 0xffff
+; SI-NEXT: s_and_b32 s5, s58, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s4, 16
+; SI-NEXT: s_or_b32 s37, s5, s7
+; SI-NEXT: s_and_b32 s7, s60, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s44, 16
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_and_b32 s9, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s52, 16
+; SI-NEXT: s_mov_b32 vcc_lo, s41
+; SI-NEXT: s_or_b32 s41, s9, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s68, 16
+; SI-NEXT: s_or_b32 s11, s11, s13
+; SI-NEXT: s_and_b32 s13, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s69, 16
+; SI-NEXT: s_or_b32 s13, s13, s15
+; SI-NEXT: s_and_b32 s15, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s70, 16
+; SI-NEXT: s_or_b32 s15, s15, s19
+; SI-NEXT: s_and_b32 s19, s39, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s71, 16
+; SI-NEXT: s_or_b32 s93, s19, s4
+; SI-NEXT: s_and_b32 s19, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s46, 16
+; SI-NEXT: s_or_b32 s43, s19, s4
+; SI-NEXT: s_and_b32 s19, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s81, 16
+; SI-NEXT: s_mov_b32 s50, s44
+; SI-NEXT: s_lshl_b32 s44, s59, 16
+; SI-NEXT: s_or_b32 s45, s19, s4
+; SI-NEXT: s_or_b32 s4, s17, s44
+; SI-NEXT: s_and_b32 s17, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s83, 16
+; SI-NEXT: s_or_b32 s47, s17, s19
+; SI-NEXT: s_mov_b32 s64, s46
+; SI-NEXT: s_lshl_b32 s46, s65, 16
+; SI-NEXT: s_and_b32 s17, s97, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 28
+; SI-NEXT: s_or_b32 s4, s17, s46
+; SI-NEXT: s_and_b32 s17, s62, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s85, 16
+; SI-NEXT: s_or_b32 s63, s17, s19
+; SI-NEXT: s_mov_b32 s66, s62
+; SI-NEXT: s_lshl_b32 s62, s98, 16
+; SI-NEXT: s_and_b32 s17, s87, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 27
+; SI-NEXT: s_or_b32 s4, s17, s62
+; SI-NEXT: s_and_b32 s17, s88, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s86, 16
+; SI-NEXT: s_or_b32 s89, s17, s19
+; SI-NEXT: s_mov_b32 s67, s88
+; SI-NEXT: s_lshl_b32 s88, s96, 16
+; SI-NEXT: s_and_b32 s17, s77, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 26
+; SI-NEXT: s_or_b32 s4, s17, s88
+; SI-NEXT: s_and_b32 s17, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s19, vcc_hi, 16
+; SI-NEXT: s_or_b32 s57, s17, s19
+; SI-NEXT: s_lshl_b32 s56, s34, 16
+; SI-NEXT: s_and_b32 s17, s55, 0xffff
+; SI-NEXT: v_writelane_b32 v62, s4, 25
+; SI-NEXT: s_or_b32 s4, s17, s56
+; SI-NEXT: v_writelane_b32 v62, s4, 24
+; SI-NEXT: s_and_b32 s19, s49, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s38, 16
+; SI-NEXT: v_readlane_b32 s8, v62, 4
+; SI-NEXT: s_or_b32 s5, s19, s4
+; SI-NEXT: s_lshl_b32 s4, s48, 16
+; SI-NEXT: s_and_b32 s19, s8, 0xffff
+; SI-NEXT: s_or_b32 s8, s19, s4
+; SI-NEXT: v_writelane_b32 v62, s8, 23
+; SI-NEXT: s_and_b32 s8, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s29, s31, 16
+; SI-NEXT: s_mov_b32 s19, s27
+; SI-NEXT: s_mov_b32 s27, s25
+; SI-NEXT: v_readlane_b32 s25, v62, 3
+; SI-NEXT: s_or_b32 s9, s8, s29
+; SI-NEXT: s_lshl_b32 s8, s54, 16
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_or_b32 s17, s25, s8
+; SI-NEXT: v_writelane_b32 v62, s17, 22
+; SI-NEXT: v_readlane_b32 s17, v62, 1
+; SI-NEXT: s_and_b32 s25, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s72, s51, 16
+; SI-NEXT: s_or_b32 s73, s25, s72
+; SI-NEXT: v_readlane_b32 s25, v62, 2
+; SI-NEXT: s_lshl_b32 s72, s53, 16
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_or_b32 s17, s25, s72
+; SI-NEXT: s_mov_b32 s25, s37
+; SI-NEXT: v_writelane_b32 v62, s24, 8
+; SI-NEXT: v_writelane_b32 v62, s25, 9
+; SI-NEXT: s_lshr_b64 s[36:37], s[36:37], 16
+; SI-NEXT: v_writelane_b32 v62, s36, 6
+; SI-NEXT: v_writelane_b32 v62, s37, 7
+; SI-NEXT: s_lshr_b64 s[36:37], s[6:7], 16
+; SI-NEXT: v_writelane_b32 v62, s36, 10
+; SI-NEXT: v_writelane_b32 v62, s37, 11
+; SI-NEXT: s_mov_b32 s25, s41
+; SI-NEXT: v_writelane_b32 v62, s24, 14
+; SI-NEXT: v_writelane_b32 v62, s25, 15
+; SI-NEXT: s_lshr_b64 s[40:41], s[40:41], 16
+; SI-NEXT: v_writelane_b32 v62, s40, 12
+; SI-NEXT: v_writelane_b32 v62, s41, 13
+; SI-NEXT: s_lshr_b64 s[40:41], s[10:11], 16
+; SI-NEXT: v_writelane_b32 v62, s40, 16
+; SI-NEXT: v_writelane_b32 v62, s41, 17
+; SI-NEXT: s_lshr_b64 s[40:41], s[12:13], 16
+; SI-NEXT: s_mov_b32 s6, s76
+; SI-NEXT: s_mov_b32 s78, s77
+; SI-NEXT: s_lshr_b64 s[76:77], s[14:15], 16
+; SI-NEXT: s_mov_b32 s14, s94
+; SI-NEXT: s_mov_b32 s41, vcc_lo
+; SI-NEXT: s_lshr_b64 s[36:37], s[44:45], 16
+; SI-NEXT: s_mov_b32 vcc_lo, s95
+; SI-NEXT: s_lshr_b64 s[94:95], s[46:47], 16
+; SI-NEXT: s_mov_b32 s10, s90
+; SI-NEXT: s_mov_b32 s12, s75
+; SI-NEXT: s_mov_b32 s77, s78
+; SI-NEXT: s_mov_b32 s90, s79
+; SI-NEXT: s_mov_b32 s75, s93
+; SI-NEXT: s_lshr_b64 s[78:79], s[92:93], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[42:43], 16
+; SI-NEXT: s_mov_b32 s42, s30
+; SI-NEXT: v_writelane_b32 v62, s36, 18
+; SI-NEXT: s_mov_b32 s95, vcc_lo
+; SI-NEXT: s_mov_b32 s30, vcc_hi
+; SI-NEXT: s_lshr_b64 vcc, s[62:63], 16
+; SI-NEXT: v_writelane_b32 v62, s37, 19
+; SI-NEXT: s_mov_b32 vcc_hi, s30
+; SI-NEXT: s_mov_b32 s36, s31
+; SI-NEXT: s_lshr_b64 s[30:31], s[88:89], 16
+; SI-NEXT: s_mov_b32 s44, s50
+; SI-NEXT: s_mov_b32 s31, s36
+; SI-NEXT: s_lshr_b64 s[36:37], s[56:57], 16
+; SI-NEXT: s_mov_b32 s56, s51
+; SI-NEXT: s_lshr_b64 s[50:51], s[4:5], 16
+; SI-NEXT: s_mov_b32 s46, s64
+; SI-NEXT: v_writelane_b32 v62, s36, 20
+; SI-NEXT: s_mov_b32 s51, s56
+; SI-NEXT: s_mov_b32 s56, s65
+; SI-NEXT: s_lshr_b64 s[64:65], s[72:73], 16
+; SI-NEXT: s_mov_b32 s79, s90
+; SI-NEXT: s_mov_b32 s62, s66
+; SI-NEXT: s_mov_b32 s88, s67
+; SI-NEXT: v_writelane_b32 v62, s37, 21
+; SI-NEXT: s_lshr_b64 s[36:37], s[8:9], 16
+; SI-NEXT: s_mov_b32 s25, s27
+; SI-NEXT: s_mov_b32 s27, s19
+; SI-NEXT: s_mov_b32 s65, s56
+; SI-NEXT: s_mov_b64 s[66:67], 0
; SI-NEXT: s_branch .LBB103_3
; SI-NEXT: .LBB103_2:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b32 s34, s4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr5
+; SI-NEXT: s_mov_b64 s[66:67], -1
+; SI-NEXT: v_writelane_b32 v62, s4, 6
+; SI-NEXT: v_writelane_b32 v62, s5, 7
+; SI-NEXT: ; implicit-def: $sgpr5
; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr82
; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr85
; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; kill: killed $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: v_writelane_b32 v62, s4, 8
+; SI-NEXT: v_writelane_b32 v62, s5, 9
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr9
+; SI-NEXT: ; kill: killed $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; kill: killed $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; kill: killed $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr17
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: v_writelane_b32 v62, s4, 10
+; SI-NEXT: v_writelane_b32 v62, s5, 11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 12
+; SI-NEXT: v_writelane_b32 v62, s5, 13
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: v_writelane_b32 v62, s4, 14
+; SI-NEXT: v_writelane_b32 v62, s5, 15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 16
+; SI-NEXT: v_writelane_b32 v62, s5, 17
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 18
+; SI-NEXT: v_writelane_b32 v62, s5, 19
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v62, s4, 20
+; SI-NEXT: v_writelane_b32 v62, s5, 21
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr5
; SI-NEXT: .LBB103_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
+; SI-NEXT: s_cselect_b32 s56, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s56, 1
; SI-NEXT: s_cbranch_scc1 .LBB103_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s52
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s49
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s48
-; SI-NEXT: v_readlane_b32 s4, v62, 7
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s21
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s37
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s35
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s34
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_readlane_b32 s4, v62, 5
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; SI-NEXT: v_or_b32_e32 v0, v1, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, vcc_hi
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s22
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s20
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, vcc_lo
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s95
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s10
+; SI-NEXT: v_or_b32_e32 v33, v2, v40
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s35
+; SI-NEXT: v_or_b32_e32 v51, v1, v9
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s12
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s94
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s93
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v51, v1
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; SI-NEXT: v_or_b32_e32 v35, v2, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s14
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, s91
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s92
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s89
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s28
+; SI-NEXT: v_or_b32_e32 v59, v1, v54
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s99
+; SI-NEXT: v_or_b32_e32 v61, v0, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s41
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v39, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s79
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s90
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s77
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s88
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v53, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s75
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v49, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s78
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v55, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s73
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v52, v0
-; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v55
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s76
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v41, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s63
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v54, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s74
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v43, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s61
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v40, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s72
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v45, v1
-; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v41
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v42, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s62
-; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v45
-; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v53
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v42
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v52
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v44, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s60
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v51
-; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v39
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s59
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v46, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s58
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v35, v1
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v46
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v40
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v49
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s57
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v34, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s56
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; SI-NEXT: v_or_b32_e32 v57, v2, v13
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s59
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v33, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s47
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v32, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s46
+; SI-NEXT: v_cvt_f16_f32_e32 v37, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s98
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s48
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v31, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s45
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s44
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s54
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s34
+; SI-NEXT: v_add_f32_e32 v32, 0x38000000, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s96
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v29, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s43
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v28, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s29
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s53
+; SI-NEXT: v_cvt_f16_f32_e32 v20, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s51
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s38
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v27, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s42
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v26, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s28
+; SI-NEXT: v_cvt_f16_f32_e32 v19, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s49
+; SI-NEXT: v_cvt_f32_f16_e32 v3, vcc_hi
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s95
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v25, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s41
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v24, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s27
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v3
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v49, v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s86
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_or_b32_e32 v16, v3, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s88
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v23, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s40
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v22, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s26
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s85
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s62
+; SI-NEXT: v_or_b32_e32 v25, v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s83
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s84
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v36, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_or_b32_e32 v23, v3, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s81
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v36
+; SI-NEXT: v_or_b32_e32 v18, v2, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s46
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v21, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s15
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v20, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s25
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v21
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s80
+; SI-NEXT: v_or_b32_e32 v27, v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s71
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s39
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_or_b32_e32 v53, v3, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s70
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT: v_or_b32_e32 v14, v2, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s27
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v19, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s14
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v18, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s69
+; SI-NEXT: v_cvt_f16_f32_e32 v34, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v46, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v34
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s25
+; SI-NEXT: v_or_b32_e32 v55, v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s23
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v43, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v46
+; SI-NEXT: v_or_b32_e32 v8, v3, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s52
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v43
+; SI-NEXT: v_or_b32_e32 v12, v2, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s26
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v17, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s44
+; SI-NEXT: v_cvt_f16_f32_e32 v45, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v56, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v45
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s60
+; SI-NEXT: v_or_b32_e32 v10, v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s58
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s42
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v58, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v4
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v36, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s23
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v28, s31
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v56
+; SI-NEXT: v_or_b32_e32 v41, v3, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s79
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v58
+; SI-NEXT: v_readlane_b32 s4, v62, 1
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_or_b32_e32 v6, v2, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
+; SI-NEXT: v_or_b32_e32 v47, v0, v52
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s65
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v28
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_readlane_b32 s4, v62, 6
-; SI-NEXT: v_cvt_f16_f32_e32 v15, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s22
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_or_b32_e32 v29, v1, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v30
+; SI-NEXT: v_or_b32_e32 v31, v2, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s77
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s87
+; SI-NEXT: v_readlane_b32 s4, v62, 4
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_readlane_b32 s4, v62, 5
-; SI-NEXT: v_cvt_f16_f32_e32 v13, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v12, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s21
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_readlane_b32 s4, v62, 4
-; SI-NEXT: v_cvt_f16_f32_e32 v11, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v32
+; SI-NEXT: v_or_b32_e32 v32, v1, v24
; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s20
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_or_b32_e32 v60, v0, v22
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s55
; SI-NEXT: v_readlane_b32 s4, v62, 3
-; SI-NEXT: v_cvt_f16_f32_e32 v9, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v37, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s19
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_readlane_b32 s4, v62, 2
-; SI-NEXT: v_cvt_f16_f32_e32 v38, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v20
+; SI-NEXT: v_or_b32_e32 v20, v2, v17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v0
-; SI-NEXT: v_readlane_b32 s4, v62, 1
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v50, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_or_b32_e32 v38, v37, v26
+; SI-NEXT: v_or_b32_e32 v37, v1, v48
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_mov_b32_e32 v1, v6
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v48, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v50
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v48
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v38
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v37
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v36
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v34
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v35
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v44
-; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v43
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v54
-; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v16
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v39, v0, v15
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
+; SI-NEXT: v_mov_b32_e32 v1, v41
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_or_b32_e32 v42, v2, v28
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[1:2], v[40:41], 16
+; SI-NEXT: v_mov_b32_e32 v2, v10
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v12
+; SI-NEXT: v_readlane_b32 s4, v62, 2
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s4
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v19
+; SI-NEXT: v_lshr_b64 v[40:41], v[28:29], 16
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v8
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v55
+; SI-NEXT: v_or_b32_e32 v50, v4, v30
+; SI-NEXT: v_lshr_b64 v[4:5], v[9:10], 16
+; SI-NEXT: v_lshr_b64 v[5:6], v[11:12], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[7:8], 16
+; SI-NEXT: v_lshr_b64 v[12:13], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[9:10], v[54:55], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v14
+; SI-NEXT: v_lshr_b64 v[13:14], v[52:53], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[26:27], 16
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v53
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v27
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v18
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v23
+; SI-NEXT: v_lshr_b64 v[18:19], v[17:18], 16
+; SI-NEXT: v_mov_b32_e32 v53, v20
+; SI-NEXT: v_mov_b32_e32 v19, v21
+; SI-NEXT: v_lshr_b64 v[20:21], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[24:25], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[15:16], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v16
+; SI-NEXT: v_mov_b32_e32 v21, v36
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v7, v56
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v11, v43
+; SI-NEXT: v_lshr_b64 v[55:56], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[43:44], v[30:31], 16
+; SI-NEXT: v_mov_b32_e32 v6, v5
+; SI-NEXT: v_mov_b32_e32 v10, v9
+; SI-NEXT: v_mov_b32_e32 v14, v13
+; SI-NEXT: v_mov_b32_e32 v30, v3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, v1
+; SI-NEXT: v_mov_b32_e32 v1, v49
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_branch .LBB103_6
; SI-NEXT: .LBB103_5:
-; SI-NEXT: v_readlane_b32 s4, v62, 8
-; SI-NEXT: v_mov_b32_e32 v25, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 9
-; SI-NEXT: v_mov_b32_e32 v24, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 10
-; SI-NEXT: v_mov_b32_e32 v23, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 11
-; SI-NEXT: v_mov_b32_e32 v22, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 12
-; SI-NEXT: v_mov_b32_e32 v21, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 13
-; SI-NEXT: v_mov_b32_e32 v20, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 14
-; SI-NEXT: v_mov_b32_e32 v19, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 15
-; SI-NEXT: v_mov_b32_e32 v18, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 16
-; SI-NEXT: v_mov_b32_e32 v17, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 17
-; SI-NEXT: v_mov_b32_e32 v50, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 18
-; SI-NEXT: v_mov_b32_e32 v15, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 19
-; SI-NEXT: v_mov_b32_e32 v14, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 20
-; SI-NEXT: v_mov_b32_e32 v13, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 21
-; SI-NEXT: v_mov_b32_e32 v12, s4
+; SI-NEXT: v_readlane_b32 s18, v62, 8
+; SI-NEXT: v_mov_b32_e32 v0, s46
+; SI-NEXT: v_readlane_b32 s19, v62, 9
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s71
+; SI-NEXT: v_mov_b32_e32 v1, s19
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: v_readlane_b32 s6, v62, 14
+; SI-NEXT: v_readlane_b32 s7, v62, 15
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: v_mov_b32_e32 v58, s4
; SI-NEXT: v_readlane_b32 s4, v62, 22
-; SI-NEXT: v_mov_b32_e32 v11, s4
+; SI-NEXT: v_mov_b32_e32 v42, s4
; SI-NEXT: v_readlane_b32 s4, v62, 23
-; SI-NEXT: v_mov_b32_e32 v10, s4
+; SI-NEXT: v_mov_b32_e32 v37, s4
; SI-NEXT: v_readlane_b32 s4, v62, 24
-; SI-NEXT: v_mov_b32_e32 v9, s4
+; SI-NEXT: v_mov_b32_e32 v39, s4
; SI-NEXT: v_readlane_b32 s4, v62, 25
-; SI-NEXT: v_mov_b32_e32 v38, s4
+; SI-NEXT: v_mov_b32_e32 v32, s4
; SI-NEXT: v_readlane_b32 s4, v62, 26
-; SI-NEXT: v_mov_b32_e32 v0, s86
-; SI-NEXT: v_mov_b32_e32 v48, s4
+; SI-NEXT: v_mov_b32_e32 v60, s4
; SI-NEXT: v_readlane_b32 s4, v62, 27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s13
-; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_mov_b32_e32 v53, s4
; SI-NEXT: v_readlane_b32 s4, v62, 28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s85
-; SI-NEXT: v_mov_b32_e32 v5, s4
+; SI-NEXT: v_mov_b32_e32 v38, s4
; SI-NEXT: v_readlane_b32 s4, v62, 29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_mov_b32_e32 v57, s4
; SI-NEXT: v_readlane_b32 s4, v62, 30
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s84
-; SI-NEXT: v_mov_b32_e32 v3, s4
+; SI-NEXT: v_mov_b32_e32 v59, s4
; SI-NEXT: v_readlane_b32 s4, v62, 31
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s83
-; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_mov_b32_e32 v61, s4
; SI-NEXT: v_readlane_b32 s4, v62, 32
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s11
-; SI-NEXT: v_mov_b32_e32 v1, s4
+; SI-NEXT: v_mov_b32_e32 v35, s4
; SI-NEXT: v_readlane_b32 s4, v62, 33
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v39, s82
-; SI-NEXT: v_mov_b32_e32 v51, s81
-; SI-NEXT: v_mov_b32_e32 v49, s10
-; SI-NEXT: v_mov_b32_e32 v53, s80
-; SI-NEXT: v_mov_b32_e32 v52, s71
-; SI-NEXT: v_mov_b32_e32 v55, s9
-; SI-NEXT: v_mov_b32_e32 v54, s70
-; SI-NEXT: v_mov_b32_e32 v41, s69
-; SI-NEXT: v_mov_b32_e32 v40, s8
-; SI-NEXT: v_mov_b32_e32 v43, s68
-; SI-NEXT: v_mov_b32_e32 v42, s67
-; SI-NEXT: v_mov_b32_e32 v45, s66
-; SI-NEXT: v_mov_b32_e32 v44, s7
-; SI-NEXT: v_mov_b32_e32 v47, s65
-; SI-NEXT: v_mov_b32_e32 v46, s6
-; SI-NEXT: v_mov_b32_e32 v57, s64
-; SI-NEXT: v_mov_b32_e32 v56, s55
-; SI-NEXT: v_mov_b32_e32 v59, s54
-; SI-NEXT: v_mov_b32_e32 v58, s53
-; SI-NEXT: v_mov_b32_e32 v61, s51
-; SI-NEXT: v_mov_b32_e32 v60, s50
-; SI-NEXT: v_mov_b32_e32 v37, s39
-; SI-NEXT: v_mov_b32_e32 v36, s38
-; SI-NEXT: v_mov_b32_e32 v33, s36
-; SI-NEXT: v_mov_b32_e32 v32, s31
-; SI-NEXT: v_mov_b32_e32 v31, s30
-; SI-NEXT: v_mov_b32_e32 v30, s99
-; SI-NEXT: v_mov_b32_e32 v29, s98
-; SI-NEXT: v_mov_b32_e32 v28, s97
-; SI-NEXT: v_mov_b32_e32 v27, s96
-; SI-NEXT: v_mov_b32_e32 v26, s87
+; SI-NEXT: v_mov_b32_e32 v51, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 34
+; SI-NEXT: v_mov_b32_e32 v33, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 35
+; SI-NEXT: v_mov_b32_e32 v52, s4
+; SI-NEXT: v_mov_b32_e32 v36, s51
+; SI-NEXT: v_mov_b32_e32 v30, s31
+; SI-NEXT: v_mov_b32_e32 v28, s38
+; SI-NEXT: v_mov_b32_e32 v27, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v24, s86
+; SI-NEXT: v_mov_b32_e32 v22, s85
+; SI-NEXT: v_mov_b32_e32 v21, s83
+; SI-NEXT: v_mov_b32_e32 v19, s81
+; SI-NEXT: v_mov_b32_e32 v34, s70
+; SI-NEXT: v_mov_b32_e32 v46, s69
+; SI-NEXT: v_mov_b32_e32 v11, s68
+; SI-NEXT: v_mov_b32_e32 v45, s52
+; SI-NEXT: v_mov_b32_e32 v7, s44
+; SI-NEXT: v_mov_b32_e32 v25, s89
+; SI-NEXT: v_mov_b32_e32 v50, s17
+; SI-NEXT: v_mov_b32_e32 v47, s61
+; SI-NEXT: v_mov_b32_e32 v29, s9
+; SI-NEXT: v_mov_b32_e32 v31, s73
+; SI-NEXT: v_mov_b32_e32 v8, s40
+; SI-NEXT: v_mov_b32_e32 v10, s76
+; SI-NEXT: v_mov_b32_e32 v12, s78
+; SI-NEXT: v_mov_b32_e32 v14, s92
+; SI-NEXT: v_mov_b32_e32 v18, s94
+; SI-NEXT: v_mov_b32_e32 v20, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v23, s30
+; SI-NEXT: v_mov_b32_e32 v55, s50
+; SI-NEXT: v_mov_b32_e32 v40, s36
+; SI-NEXT: v_mov_b32_e32 v43, s64
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: .LBB103_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v48
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s11
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s15
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s75
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
+; SI-NEXT: v_mov_b32_e32 v1, s43
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s45
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v13
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, s47
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s63
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s57
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_readlane_b32 s4, v62, 6
+; SI-NEXT: v_readlane_b32 s5, v62, 7
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[33:34], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v38
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v60
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v46
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v44
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v42
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v52
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 10
+; SI-NEXT: v_readlane_b32 s5, v62, 11
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 12
+; SI-NEXT: v_readlane_b32 s5, v62, 13
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 16
+; SI-NEXT: v_readlane_b32 s5, v62, 17
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 18
+; SI-NEXT: v_readlane_b32 s5, v62, 19
+; SI-NEXT: v_mov_b32_e32 v54, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 20
+; SI-NEXT: v_readlane_b32 s5, v62, 21
+; SI-NEXT: v_mov_b32_e32 v26, s4
+; SI-NEXT: .LBB103_6: ; %end
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v52
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v58
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v45
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v34
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v46
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v38
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v50
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
; SI-NEXT: v_readlane_b32 s30, v63, 34
; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: v_readlane_b32 s99, v63, 33
@@ -218351,46 +213751,109 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v16
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v33
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v3, v3, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v51
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v4, v5, v4
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[4:5], v[5:6], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v52
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v5, v5, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v35
+; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v52
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v7, v7, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v61
+; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v54
+; SI-NEXT: v_or_b32_e32 v16, v17, v16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v52
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v9, v9, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v10, v11, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v52
+; SI-NEXT: v_or_b32_e32 v11, v11, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v57
+; SI-NEXT: v_or_b32_e32 v12, v13, v12
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v34
+; SI-NEXT: v_or_b32_e32 v13, v13, v33
+; SI-NEXT: v_and_b32_e32 v33, 0xffff, v47
+; SI-NEXT: v_or_b32_e32 v14, v33, v14
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v34
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v17, v17, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v53
+; SI-NEXT: v_or_b32_e32 v18, v19, v18
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v34
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v19, v19, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v60
+; SI-NEXT: v_or_b32_e32 v20, v21, v20
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v34
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v23
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v39
+; SI-NEXT: v_or_b32_e32 v24, v24, v26
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v22, v22, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[24:25], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[26:27], 16
-; SI-NEXT: v_lshr_b64 v[14:15], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[30:31], 16
-; SI-NEXT: v_lshr_b64 v[16:17], v[17:18], 16
-; SI-NEXT: v_lshr_b64 v[17:18], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[60:61], 16
-; SI-NEXT: v_lshr_b64 v[19:20], v[58:59], 16
-; SI-NEXT: v_lshr_b64 v[20:21], v[56:57], 16
-; SI-NEXT: v_lshr_b64 v[21:22], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[42:43], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[40:41], 16
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v40
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v27
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v37
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v55
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v33
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v42
+; SI-NEXT: v_or_b32_e32 v28, v28, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v43
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -218405,32 +213868,12 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[25:26], v[54:55], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[38:39], 16
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[30:31], v[36:37], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[31:32], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v0, v33
-; SI-NEXT: v_mov_b32_e32 v1, v34
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v36
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -218981,1110 +214424,439 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v19
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v21
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v21
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v18
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v29
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v25
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v30
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v28
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v27
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v23
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v20
-; SI-NEXT: v_and_b32_e32 v63, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v32
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v11
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v34
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v13
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v18
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v5
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v41
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v7
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v36
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v29
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v9
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v17
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v25
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v13
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v11
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v9
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v7
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v5
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v3
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v62
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v38
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v40
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v35
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v33
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v63
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v61
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v60
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v43
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v42
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v59
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v45
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v44
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v47
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v46
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v57
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v56
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v55
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v54
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v53
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v51
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v49
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v48
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v23
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; kill: killed $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; kill: killed $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; kill: killed $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v36
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v31
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB104_2
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v38
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v21
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; kill: killed $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; kill: killed $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v32
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; kill: killed $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v37
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v36
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v42
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v35
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v46
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v30
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v26
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v24
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v61
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v60
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v22
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v18
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v63
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v62
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: .LBB104_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB104_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v38
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v19
-; SI-NEXT: v_alignbit_b32 v0, v15, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v7
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v2
+; SI-NEXT: s_cbranch_execz .LBB104_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v63
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v63, 0x40c00000, v63
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v41
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v37
+; SI-NEXT: v_mov_b32_e32 v55, v54
+; SI-NEXT: v_mov_b32_e32 v54, v53
+; SI-NEXT: v_mov_b32_e32 v53, v52
+; SI-NEXT: v_mov_b32_e32 v52, v51
+; SI-NEXT: v_mov_b32_e32 v51, v50
+; SI-NEXT: v_mov_b32_e32 v50, v49
+; SI-NEXT: v_mov_b32_e32 v49, v48
+; SI-NEXT: v_mov_b32_e32 v48, v39
+; SI-NEXT: v_mov_b32_e32 v39, v38
+; SI-NEXT: v_mov_b32_e32 v38, v37
+; SI-NEXT: v_mov_b32_e32 v37, v35
+; SI-NEXT: v_mov_b32_e32 v35, v34
+; SI-NEXT: v_mov_b32_e32 v34, v33
+; SI-NEXT: v_mov_b32_e32 v33, v40
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v63
+; SI-NEXT: v_alignbit_b32 v0, v40, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v62
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_add_f32_e32 v62, 0x40c00000, v40
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; SI-NEXT: v_alignbit_b32 v0, v4, v0, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v62
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
+; SI-NEXT: v_alignbit_b32 v0, v40, v2, 16
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v61
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_add_f32_e32 v61, 0x40c00000, v40
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v6, v0, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v61
+; SI-NEXT: v_alignbit_b32 v2, v40, v4, 16
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v60
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v40
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; SI-NEXT: v_alignbit_b32 v0, v11, v0, 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v11
-; SI-NEXT: v_alignbit_b32 v0, v13, v0, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v10
-; SI-NEXT: v_alignbit_b32 v0, v12, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v35
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v60
+; SI-NEXT: v_alignbit_b32 v4, v40, v6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v59
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_add_f32_e32 v59, 0x40c00000, v40
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v12, v0, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v59
+; SI-NEXT: v_alignbit_b32 v6, v40, v8, 16
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v58
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_add_f32_e32 v58, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v58
+; SI-NEXT: v_alignbit_b32 v8, v40, v10, 16
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v57
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v40
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v56
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_alignbit_b32 v0, v13, v0, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_alignbit_b32 v0, v14, v0, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v40
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v56
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; SI-NEXT: v_alignbit_b32 v10, v57, v12, 16
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_alignbit_b32 v0, v15, v0, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_alignbit_b32 v0, v16, v0, 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v62
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v18, v16, v0, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v20, v16, v15, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v58
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v21, v16, v15, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v56
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v22, v16, v15, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v46
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v24, v16, v15, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v44
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v26, v16, v15, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v42
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: v_alignbit_b32 v27, v16, v15, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_alignbit_b32 v0, v18, v0, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_alignbit_b32 v33, v50, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_alignbit_b32 v34, v51, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v56
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_add_f32_e32 v47, 0x40c00000, v47
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; SI-NEXT: v_alignbit_b32 v12, v57, v14, 16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_alignbit_b32 v25, v52, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_alignbit_b32 v28, v39, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v47
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_add_f32_e32 v46, 0x40c00000, v46
+; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; SI-NEXT: v_alignbit_b32 v14, v57, v16, 16
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v46
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_add_f32_e32 v45, 0x40c00000, v45
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; SI-NEXT: v_alignbit_b32 v16, v57, v18, 16
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v45
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_add_f32_e32 v44, 0x40c00000, v44
+; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; SI-NEXT: v_alignbit_b32 v18, v57, v20, 16
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v44
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_add_f32_e32 v43, 0x40c00000, v43
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; SI-NEXT: v_alignbit_b32 v20, v57, v22, 16
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v43
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_add_f32_e32 v42, 0x40c00000, v42
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; SI-NEXT: v_alignbit_b32 v22, v57, v24, 16
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v42
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_add_f32_e32 v41, 0x40c00000, v41
+; SI-NEXT: v_alignbit_b32 v24, v57, v26, 16
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v41
+; SI-NEXT: v_alignbit_b32 v26, v57, v28, 16
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_add_f32_e32 v57, 0x40c00000, v57
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v57
+; SI-NEXT: v_alignbit_b32 v28, v32, v30, 16
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_alignbit_b32 v23, v53, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_alignbit_b32 v31, v48, v15, 16
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_alignbit_b32 v40, v54, v16, 16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: v_alignbit_b32 v1, v40, v1, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v23, v3, 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v25, v5, 16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v34, v7, 16
-; SI-NEXT: v_alignbit_b32 v15, v49, v15, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v33, v9, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v15, v2, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v31, v4, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_alignbit_b32 v17, v55, v16, 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v19
-; SI-NEXT: v_mov_b32_e32 v19, v28
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v41
+; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v42
+; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v43
+; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v44
+; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v46
+; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v47
+; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v56
+; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v40
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v30
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v32
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v31, v30, v31, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v19, v6, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_alignbit_b32 v1, v27, v11, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_alignbit_b32 v1, v26, v10, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_alignbit_b32 v1, v24, v8, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_alignbit_b32 v1, v22, v12, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_alignbit_b32 v1, v21, v13, 16
-; SI-NEXT: v_alignbit_b32 v16, v17, v16, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_alignbit_b32 v1, v20, v14, 16
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: .LBB104_4: ; %end
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v57
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v33
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v34
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v38
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v48
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v49
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v50
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v51
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v52
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v53
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v54
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v55
+; SI-NEXT: v_add_f32_e32 v57, 0x40c00000, v57
+; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
+; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
+; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v35
+; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v37
+; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v38
+; SI-NEXT: v_add_f32_e32 v39, 0x40c00000, v39
+; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v48
+; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v49
+; SI-NEXT: v_add_f32_e32 v50, 0x40c00000, v50
+; SI-NEXT: v_add_f32_e32 v51, 0x40c00000, v51
+; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v53, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v54, 0x40c00000, v54
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v57
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v34
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v37
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v38
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v39
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v48
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v49
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v50
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v51
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v52
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v54
+; SI-NEXT: v_alignbit_b32 v29, v36, v29, 16
+; SI-NEXT: v_alignbit_b32 v27, v33, v27, 16
+; SI-NEXT: v_alignbit_b32 v25, v34, v25, 16
+; SI-NEXT: v_alignbit_b32 v23, v35, v23, 16
+; SI-NEXT: v_alignbit_b32 v21, v37, v21, 16
+; SI-NEXT: v_alignbit_b32 v19, v38, v19, 16
+; SI-NEXT: v_alignbit_b32 v17, v39, v17, 16
+; SI-NEXT: v_alignbit_b32 v15, v48, v15, 16
+; SI-NEXT: v_alignbit_b32 v13, v49, v13, 16
+; SI-NEXT: v_alignbit_b32 v11, v50, v11, 16
+; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v58
+; SI-NEXT: v_alignbit_b32 v9, v51, v9, 16
+; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v59
+; SI-NEXT: v_alignbit_b32 v7, v52, v7, 16
+; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v60
+; SI-NEXT: v_alignbit_b32 v5, v53, v5, 16
+; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v61
+; SI-NEXT: v_alignbit_b32 v3, v54, v3, 16
+; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v62
+; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v63
+; SI-NEXT: v_alignbit_b32 v56, v15, v56, 16
+; SI-NEXT: v_alignbit_b32 v47, v17, v47, 16
+; SI-NEXT: v_alignbit_b32 v46, v19, v46, 16
+; SI-NEXT: v_alignbit_b32 v45, v21, v45, 16
+; SI-NEXT: v_alignbit_b32 v44, v23, v44, 16
+; SI-NEXT: v_alignbit_b32 v43, v25, v43, 16
+; SI-NEXT: v_alignbit_b32 v42, v27, v42, 16
+; SI-NEXT: v_alignbit_b32 v41, v29, v41, 16
+; SI-NEXT: v_alignbit_b32 v32, v31, v32, 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v30
+; SI-NEXT: v_mov_b32_e32 v30, v28
+; SI-NEXT: v_mov_b32_e32 v28, v26
+; SI-NEXT: v_mov_b32_e32 v26, v24
+; SI-NEXT: v_mov_b32_e32 v24, v22
+; SI-NEXT: v_mov_b32_e32 v22, v20
+; SI-NEXT: v_mov_b32_e32 v20, v18
+; SI-NEXT: v_mov_b32_e32 v18, v16
+; SI-NEXT: v_mov_b32_e32 v16, v14
+; SI-NEXT: v_mov_b32_e32 v14, v12
+; SI-NEXT: v_mov_b32_e32 v12, v10
+; SI-NEXT: v_mov_b32_e32 v10, v8
+; SI-NEXT: v_mov_b32_e32 v8, v6
+; SI-NEXT: v_mov_b32_e32 v6, v4
+; SI-NEXT: v_mov_b32_e32 v4, v2
+; SI-NEXT: v_mov_b32_e32 v2, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v55, 0x40c00000, v55
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
+; SI-NEXT: v_alignbit_b32 v1, v55, v1, 16
+; SI-NEXT: v_alignbit_b32 v63, v1, v62, 16
+; SI-NEXT: v_alignbit_b32 v62, v3, v61, 16
+; SI-NEXT: v_alignbit_b32 v61, v5, v60, 16
+; SI-NEXT: v_alignbit_b32 v60, v7, v59, 16
+; SI-NEXT: v_alignbit_b32 v59, v9, v58, 16
+; SI-NEXT: v_alignbit_b32 v58, v11, v57, 16
+; SI-NEXT: v_alignbit_b32 v57, v13, v40, 16
+; SI-NEXT: v_mov_b32_e32 v40, v36
+; SI-NEXT: .LBB104_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v55
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v54
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v53
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v52
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v51
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v50
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v49
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v48
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v39
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v36, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v63
+; SI-NEXT: v_or_b32_e32 v0, v0, v32
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v55
+; SI-NEXT: v_or_b32_e32 v1, v1, v32
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v40
-; SI-NEXT: v_or_b32_e32 v3, v3, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v62
+; SI-NEXT: v_or_b32_e32 v2, v2, v32
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v54
+; SI-NEXT: v_or_b32_e32 v3, v3, v32
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v23
-; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v61
+; SI-NEXT: v_or_b32_e32 v4, v4, v32
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v53
+; SI-NEXT: v_or_b32_e32 v5, v5, v32
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_or_b32_e32 v6, v6, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v25
-; SI-NEXT: v_or_b32_e32 v7, v7, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v60
+; SI-NEXT: v_or_b32_e32 v6, v6, v32
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v52
+; SI-NEXT: v_or_b32_e32 v7, v7, v32
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v34
-; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v59
+; SI-NEXT: v_or_b32_e32 v8, v8, v32
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v51
+; SI-NEXT: v_or_b32_e32 v9, v9, v32
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_or_b32_e32 v10, v10, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v33
-; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v58
+; SI-NEXT: v_or_b32_e32 v10, v10, v32
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v50
+; SI-NEXT: v_or_b32_e32 v11, v11, v32
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_or_b32_e32 v12, v12, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v15
-; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v57
+; SI-NEXT: v_or_b32_e32 v12, v12, v32
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v49
+; SI-NEXT: v_or_b32_e32 v13, v13, v32
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v14, v14, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v31
-; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v56
+; SI-NEXT: v_or_b32_e32 v14, v14, v32
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v48
+; SI-NEXT: v_or_b32_e32 v15, v15, v32
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT: v_or_b32_e32 v16, v16, v17
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v19
-; SI-NEXT: v_or_b32_e32 v17, v17, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v47
+; SI-NEXT: v_or_b32_e32 v16, v16, v32
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v39
+; SI-NEXT: v_or_b32_e32 v17, v17, v32
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v18, v18, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v46
+; SI-NEXT: v_or_b32_e32 v18, v18, v32
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT: v_or_b32_e32 v19, v19, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v38
+; SI-NEXT: v_or_b32_e32 v19, v19, v32
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: v_or_b32_e32 v20, v20, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v45
+; SI-NEXT: v_or_b32_e32 v20, v20, v32
; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT: v_or_b32_e32 v21, v21, v22
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; SI-NEXT: v_or_b32_e32 v21, v21, v32
; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT: v_or_b32_e32 v22, v22, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v44
+; SI-NEXT: v_or_b32_e32 v22, v22, v32
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v35
+; SI-NEXT: v_or_b32_e32 v23, v23, v32
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT: v_or_b32_e32 v24, v24, v25
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v43
+; SI-NEXT: v_or_b32_e32 v24, v24, v32
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v34
+; SI-NEXT: v_or_b32_e32 v25, v25, v32
; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT: v_or_b32_e32 v26, v26, v27
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v42
+; SI-NEXT: v_or_b32_e32 v26, v26, v32
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; SI-NEXT: v_or_b32_e32 v27, v27, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
+; SI-NEXT: v_or_b32_e32 v27, v27, v32
; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; SI-NEXT: v_or_b32_e32 v28, v28, v29
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v41
+; SI-NEXT: v_or_b32_e32 v28, v28, v32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v40
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT: v_or_b32_e32 v29, v29, v30
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v29, v29, v32
; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; SI-NEXT: v_or_b32_e32 v30, v30, v31
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v36
+; SI-NEXT: v_or_b32_e32 v30, v30, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -222302,7 +217074,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
@@ -222322,999 +217094,550 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v63, s34, 0
; SI-NEXT: v_writelane_b32 v63, s35, 1
; SI-NEXT: v_writelane_b32 v63, s36, 2
-; SI-NEXT: v_writelane_b32 v63, s30, 3
-; SI-NEXT: v_writelane_b32 v63, s31, 4
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: v_readfirstlane_b32 s60, v12
-; SI-NEXT: v_readfirstlane_b32 s62, v11
-; SI-NEXT: v_readfirstlane_b32 s72, v10
-; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: v_readfirstlane_b32 s76, v8
-; SI-NEXT: v_readfirstlane_b32 s78, v7
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s62, 16
-; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s72, 16
-; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
-; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s92, 16
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
-; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v18
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s23
-; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s41
-; SI-NEXT: s_waitcnt expcnt(5)
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s29
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s27
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s25
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s30
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s94
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s79
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s75
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s61
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s59
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s57
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s47
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s42
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s43
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v63, s37, 3
+; SI-NEXT: v_writelane_b32 v63, s30, 4
+; SI-NEXT: v_writelane_b32 v63, s31, 5
+; SI-NEXT: v_readfirstlane_b32 s57, v17
+; SI-NEXT: v_readfirstlane_b32 s74, v16
+; SI-NEXT: v_readfirstlane_b32 s47, v15
+; SI-NEXT: v_readfirstlane_b32 s76, v14
+; SI-NEXT: v_readfirstlane_b32 s45, v13
+; SI-NEXT: v_readfirstlane_b32 s78, v12
+; SI-NEXT: v_readfirstlane_b32 s44, v11
+; SI-NEXT: v_readfirstlane_b32 s79, v10
+; SI-NEXT: v_readfirstlane_b32 s42, v9
+; SI-NEXT: v_readfirstlane_b32 s90, v8
+; SI-NEXT: v_readfirstlane_b32 s40, v7
+; SI-NEXT: v_readfirstlane_b32 s93, v6
+; SI-NEXT: v_readfirstlane_b32 s15, v5
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s14, v3
+; SI-NEXT: v_readfirstlane_b32 s31, v2
+; SI-NEXT: v_readfirstlane_b32 s13, v1
+; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: s_lshr_b32 s12, s29, 16
+; SI-NEXT: s_lshr_b32 s89, s28, 16
+; SI-NEXT: s_lshr_b32 s11, s27, 16
+; SI-NEXT: s_lshr_b32 s92, s26, 16
+; SI-NEXT: s_lshr_b32 s10, s25, 16
+; SI-NEXT: s_lshr_b32 s94, s24, 16
+; SI-NEXT: s_lshr_b32 s9, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s8, s21, 16
+; SI-NEXT: s_lshr_b32 s34, s20, 16
+; SI-NEXT: s_lshr_b32 s7, s19, 16
+; SI-NEXT: s_lshr_b32 s36, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s37, s16, 16
+; SI-NEXT: s_lshr_b32 s72, s57, 16
+; SI-NEXT: s_lshr_b32 s60, s74, 16
+; SI-NEXT: s_lshr_b32 s62, s47, 16
+; SI-NEXT: s_lshr_b32 s63, s76, 16
+; SI-NEXT: s_lshr_b32 s61, s45, 16
+; SI-NEXT: s_lshr_b32 s73, s78, 16
+; SI-NEXT: s_lshr_b32 s59, s44, 16
+; SI-NEXT: s_lshr_b32 s75, s79, 16
+; SI-NEXT: s_lshr_b32 s58, s42, 16
+; SI-NEXT: s_lshr_b32 s77, s90, 16
+; SI-NEXT: s_lshr_b32 s56, s40, 16
+; SI-NEXT: s_lshr_b32 s88, s93, 16
+; SI-NEXT: s_lshr_b32 s46, s15, 16
+; SI-NEXT: s_lshr_b32 s91, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s43, s14, 16
+; SI-NEXT: s_lshr_b32 s95, s31, 16
+; SI-NEXT: s_lshr_b32 s41, s13, 16
+; SI-NEXT: s_lshr_b32 s30, s35, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v18
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v27, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v42
-; SI-NEXT: v_mov_b32_e32 v48, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v43
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v44
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v61
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v33
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v28
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v56
-; SI-NEXT: v_mov_b32_e32 v56, v59
-; SI-NEXT: v_mov_b32_e32 v59, v27
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v35
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v37
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v39
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v36
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v55
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v14
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v32, v48
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v60
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v23
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v52
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v49
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v53
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v32
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
+; SI-NEXT: s_branch .LBB105_3
+; SI-NEXT: .LBB105_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB105_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB105_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v23, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s37, 16
+; SI-NEXT: v_add_f32_e32 v28, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v28
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s18, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v30
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s36, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v40
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s20, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s34, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v18, v0
-; SI-NEXT: s_branch .LBB105_3
-; SI-NEXT: .LBB105_2:
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v21
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s22, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
+; SI-NEXT: v_add_f32_e32 v19, s4, v23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v19
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s94, 16
+; SI-NEXT: v_add_f32_e32 v17, s4, v23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s92, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v15
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s89, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: .LBB105_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: v_mov_b32_e32 v24, v53
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_cbranch_scc1 .LBB105_5
-; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s35, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v61, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v59
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v42, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s31, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s95, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, vcc_lo, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[59:60], v[33:34], 16
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v45, 0x40c00000, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v51, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v51
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s91, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s93, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s88, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v23
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s90, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s77, 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s79, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v24
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s78, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v25
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s76, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s63, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v15
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s74, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s60, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v13
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s57, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s72, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s47, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s62, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v11
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v6
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s45, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s61, 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v5
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v10
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s4, s44, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v26, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s59, 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v3
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v12
-; SI-NEXT: v_lshr_b64 v[54:55], v[8:9], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v14
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v27
+; SI-NEXT: s_lshl_b32 s4, s42, 16
+; SI-NEXT: v_add_f32_e32 v32, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s58, 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e32 v1, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: v_add_f32_e32 v48, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s56, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v47, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s46, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v1
+; SI-NEXT: v_lshr_b64 v[51:52], v[48:49], 16
+; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v7
+; SI-NEXT: v_lshr_b64 v[41:42], v[32:33], 16
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v4
+; SI-NEXT: v_lshr_b64 v[53:54], v[47:48], 16
+; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v0, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s43, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_lshr_b64 v[7:8], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s41, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v9
+; SI-NEXT: v_lshr_b64 v[9:10], v[2:3], 16
+; SI-NEXT: v_add_f32_e32 v42, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v10
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
+; SI-NEXT: v_lshr_b64 v[11:12], v[42:43], 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v13
+; SI-NEXT: v_lshr_b64 v[13:14], v[4:5], 16
+; SI-NEXT: v_add_f32_e32 v36, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v14, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v14
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v15
+; SI-NEXT: v_lshr_b64 v[15:16], v[36:37], 16
+; SI-NEXT: v_add_f32_e32 v44, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_add_f32_e32 v16, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v17
+; SI-NEXT: v_lshr_b64 v[17:18], v[44:45], 16
+; SI-NEXT: v_add_f32_e32 v46, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v18, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v18
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v19
+; SI-NEXT: v_lshr_b64 v[19:20], v[46:47], 16
+; SI-NEXT: v_add_f32_e32 v55, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v20, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v20
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v21
+; SI-NEXT: v_lshr_b64 v[21:22], v[55:56], 16
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v24
+; SI-NEXT: v_lshr_b64 v[38:39], v[20:21], 16
+; SI-NEXT: v_mov_b32_e32 v20, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v57, s4, v23
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v22, s4, v23
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v22
+; SI-NEXT: v_lshr_b64 v[23:24], v[57:58], 16
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v28
+; SI-NEXT: v_lshr_b64 v[27:28], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[18:19], 16
; SI-NEXT: v_lshr_b64 v[29:30], v[8:9], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v55, v2
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v44, v29
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v16
-; SI-NEXT: v_lshr_b64 v[39:40], v[8:9], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v37
-; SI-NEXT: v_lshr_b64 v[47:48], v[8:9], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v17
-; SI-NEXT: v_lshr_b64 v[35:36], v[8:9], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; SI-NEXT: v_mov_b32_e32 v17, v56
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v14
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v18
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[30:31], v[6:7], 16
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v18
-; SI-NEXT: v_mov_b32_e32 v18, v57
-; SI-NEXT: v_mov_b32_e32 v56, v3
-; SI-NEXT: v_mov_b32_e32 v3, v27
-; SI-NEXT: v_mov_b32_e32 v2, v26
-; SI-NEXT: v_lshr_b64 v[27:28], v[30:31], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v37
-; SI-NEXT: v_lshr_b64 v[37:38], v[52:53], 16
-; SI-NEXT: v_mov_b32_e32 v28, v27
+; SI-NEXT: v_lshr_b64 v[30:31], v[52:53], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[25:26], 16
+; SI-NEXT: v_lshr_b64 v[59:60], v[14:15], 16
+; SI-NEXT: v_mov_b32_e32 v39, v33
+; SI-NEXT: v_mov_b32_e32 v14, v41
+; SI-NEXT: v_lshr_b64 v[32:33], v[40:41], 16
+; SI-NEXT: v_mov_b32_e32 v25, v30
+; SI-NEXT: v_lshr_b64 v[61:62], v[16:17], 16
+; SI-NEXT: v_mov_b32_e32 v16, v43
+; SI-NEXT: v_lshr_b64 v[34:35], v[12:13], 16
+; SI-NEXT: v_lshr_b64 v[35:36], v[10:11], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[50:51], 16
+; SI-NEXT: v_mov_b32_e32 v36, v26
+; SI-NEXT: v_mov_b32_e32 v22, v5
+; SI-NEXT: v_mov_b32_e32 v18, v3
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v19
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v19
+; SI-NEXT: v_mov_b32_e32 v30, v1
+; SI-NEXT: v_lshr_b64 v[40:41], v[0:1], 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v20
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v8, v50
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v34
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v20
-; SI-NEXT: v_mov_b32_e32 v9, v3
-; SI-NEXT: v_lshr_b64 v[49:50], v[49:50], 16
-; SI-NEXT: v_lshr_b64 v[33:34], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v50, v39
-; SI-NEXT: v_lshr_b64 v[29:30], v[26:27], 16
-; SI-NEXT: v_mov_b32_e32 v34, v33
-; SI-NEXT: v_mov_b32_e32 v26, v35
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v31, v1
+; SI-NEXT: v_lshr_b64 v[43:44], v[0:1], 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshr_b64 v[41:42], v[0:1], 16
+; SI-NEXT: v_mov_b32_e32 v33, v1
+; SI-NEXT: s_branch .LBB105_6
+; SI-NEXT: .LBB105_5:
+; SI-NEXT: v_mov_b32_e32 v1, s59
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v62
-; SI-NEXT: v_lshr_b64 v[61:62], v[61:62], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, s61
+; SI-NEXT: v_mov_b32_e32 v58, s6
+; SI-NEXT: v_mov_b32_e32 v56, s7
+; SI-NEXT: v_mov_b32_e32 v47, s8
+; SI-NEXT: v_mov_b32_e32 v45, s9
+; SI-NEXT: v_mov_b32_e32 v37, s10
+; SI-NEXT: v_mov_b32_e32 v22, s11
+; SI-NEXT: v_mov_b32_e32 v16, s12
+; SI-NEXT: v_mov_b32_e32 v18, s41
+; SI-NEXT: v_mov_b32_e32 v20, s43
+; SI-NEXT: v_mov_b32_e32 v48, s46
+; SI-NEXT: v_mov_b32_e32 v49, s56
+; SI-NEXT: v_mov_b32_e32 v39, s58
+; SI-NEXT: v_mov_b32_e32 v27, s37
+; SI-NEXT: v_mov_b32_e32 v38, s36
+; SI-NEXT: v_mov_b32_e32 v28, s34
+; SI-NEXT: v_mov_b32_e32 v61, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v59, s94
+; SI-NEXT: v_mov_b32_e32 v34, s92
+; SI-NEXT: v_mov_b32_e32 v35, s89
+; SI-NEXT: v_mov_b32_e32 v29, s30
+; SI-NEXT: v_mov_b32_e32 v25, s91
+; SI-NEXT: v_mov_b32_e32 v54, s88
+; SI-NEXT: v_mov_b32_e32 v32, s77
+; SI-NEXT: v_mov_b32_e32 v24, s75
+; SI-NEXT: v_mov_b32_e32 v40, s73
+; SI-NEXT: v_mov_b32_e32 v43, s63
+; SI-NEXT: v_mov_b32_e32 v41, s60
+; SI-NEXT: v_mov_b32_e32 v23, s17
+; SI-NEXT: v_mov_b32_e32 v21, s19
+; SI-NEXT: v_mov_b32_e32 v19, s21
+; SI-NEXT: v_mov_b32_e32 v17, s23
+; SI-NEXT: v_mov_b32_e32 v15, s25
+; SI-NEXT: v_mov_b32_e32 v13, s27
+; SI-NEXT: v_mov_b32_e32 v11, s29
+; SI-NEXT: v_mov_b32_e32 v9, s13
+; SI-NEXT: v_mov_b32_e32 v7, s14
+; SI-NEXT: v_mov_b32_e32 v53, s15
+; SI-NEXT: v_mov_b32_e32 v51, s40
+; SI-NEXT: v_mov_b32_e32 v14, s42
+; SI-NEXT: v_mov_b32_e32 v36, s44
+; SI-NEXT: v_mov_b32_e32 v30, s45
+; SI-NEXT: v_mov_b32_e32 v31, s47
+; SI-NEXT: v_mov_b32_e32 v33, s57
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v43
-; SI-NEXT: v_lshr_b64 v[42:43], v[42:43], 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[38:39], v[41:42], 16
-; SI-NEXT: v_lshr_b64 v[39:40], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v60, v59
-; SI-NEXT: v_lshr_b64 v[58:59], v[58:59], 16
-; SI-NEXT: v_mov_b32_e32 v43, v42
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v20, v7
-; SI-NEXT: v_mov_b32_e32 v19, v6
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v62, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v10
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, s62
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[2:3], 16
-; SI-NEXT: v_mov_b32_e32 v2, v56
+; SI-NEXT: v_mov_b32_e32 v1, s72
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s74
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s76
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s78
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s79
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, v5
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v11
-; SI-NEXT: v_lshr_b64 v[11:12], v[55:56], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v13
-; SI-NEXT: v_lshr_b64 v[13:14], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v15
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s90
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s93
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, vcc_lo
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v55, v1
-; SI-NEXT: v_lshr_b64 v[15:16], v[0:1], 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v20
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v23
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v25
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s28
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s26
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v51
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[21:22], 16
-; SI-NEXT: v_mov_b32_e32 v21, v46
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[45:46], 16
-; SI-NEXT: v_mov_b32_e32 v45, v1
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[22:23], v[32:33], 16
-; SI-NEXT: v_mov_b32_e32 v46, v54
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v52, v53
-; SI-NEXT: v_lshr_b64 v[56:57], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v57, v37
-; SI-NEXT: v_lshr_b64 v[0:1], v[20:21], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[18:19], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[16:17], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[14:15], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[12:13], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[10:11], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[8:9], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[36:37], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v36, v49
-; SI-NEXT: v_lshr_b64 v[48:49], v[48:49], 16
-; SI-NEXT: v_mov_b32_e32 v41, v0
+; SI-NEXT: v_mov_b32_e32 v0, s24
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, v61
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: .LBB105_5: ; %end
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v0, s22
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v56
+; SI-NEXT: v_mov_b32_e32 v0, s20
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s18
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v0, s95
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: .LBB105_6: ; %end
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v45
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v27
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v23
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v21
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v19
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v17
@@ -223323,174 +217646,149 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v9
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v13
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v11
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v57
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v21, 0xffff, v34
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v28
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s30, v63, 3
-; SI-NEXT: v_readlane_b32 s31, v63, 4
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v35
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v53
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v51
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_readlane_b32 s30, v63, 4
+; SI-NEXT: v_readlane_b32 s31, v63, 5
+; SI-NEXT: v_readlane_b32 s37, v63, 3
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v58
; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v38
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v56
; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v41
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v28
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v47
; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v61
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: v_or_b32_e32 v6, v6, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v45
; SI-NEXT: v_or_b32_e32 v7, v8, v7
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v59
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v55
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v37
; SI-NEXT: v_or_b32_e32 v9, v10, v9
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v34
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v22
; SI-NEXT: v_or_b32_e32 v11, v12, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v51
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v12, v12, v14
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v16
; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v54
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v18
; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v46
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v27
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v26
; SI-NEXT: v_or_b32_e32 v16, v16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v62
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v17, v17, v18
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v44
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v27
; SI-NEXT: v_or_b32_e32 v18, v18, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v52
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v48
; SI-NEXT: v_or_b32_e32 v19, v19, v20
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v50
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v20, v20, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v49
; SI-NEXT: v_or_b32_e32 v21, v21, v22
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v47
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v29
-; SI-NEXT: v_or_b32_e32 v22, v22, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v25
-; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: v_and_b32_e32 v24, 0xffff, v26
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v38
-; SI-NEXT: v_or_b32_e32 v24, v24, v25
-; SI-NEXT: v_and_b32_e32 v25, 0xffff, v43
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v31
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v48
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v32
+; SI-NEXT: v_or_b32_e32 v22, v22, v25
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v39
+; SI-NEXT: v_or_b32_e32 v23, v23, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v24, v25, v24
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v36
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v27
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; SI-NEXT: v_or_b32_e32 v26, v26, v27
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v40
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v30
; SI-NEXT: v_or_b32_e32 v27, v27, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v58
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v43
; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
; SI-NEXT: v_or_b32_e32 v28, v28, v29
-; SI-NEXT: v_and_b32_e32 v29, 0xffff, v60
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v31
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v31
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v41
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -223506,17 +217804,16 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v29, v29, v30
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
; SI-NEXT: v_or_b32_e32 v30, v30, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v36
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v33
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -226220,252 +220517,448 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v4
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v6
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v49
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v48
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v10
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v39
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v12
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v38
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v28
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v36
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v58
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v1
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v24
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v58
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v60
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v5
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v63
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v7
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v60
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v9
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v40
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v11
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v54
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v13
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v51
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v15
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v56
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v47
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v14
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v52
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v37
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v46
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v48
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v52
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v54
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
-; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
-; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v43
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
-; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v45
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
-; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v56
-; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v57
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v58
-; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v59
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v60
-; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v61
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v63
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v62
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; kill: killed $vgpr31
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v33
+; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v59
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v57
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v44
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v43
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v42
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v41
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v55
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v53
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v52
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; kill: killed $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB106_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v58
+; SI-NEXT: v_or_b32_e32 v61, v1, v50
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v63
+; SI-NEXT: v_or_b32_e32 v47, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v60
+; SI-NEXT: v_or_b32_e32 v62, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v40
+; SI-NEXT: v_or_b32_e32 v59, v1, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v54
+; SI-NEXT: v_or_b32_e32 v57, v1, v3
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v51
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_alignbit_b32 v1, v61, v5, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: v_or_b32_e32 v56, v1, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_alignbit_b32 v1, v47, v7, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_or_b32_e32 v46, v1, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v62, v9, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_or_b32_e32 v44, v1, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v1, v59, v11, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_or_b32_e32 v43, v1, v3
+; SI-NEXT: v_alignbit_b32 v1, v57, v13, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: v_or_b32_e32 v42, v1, v48
+; SI-NEXT: v_alignbit_b32 v1, v56, v15, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v21
+; SI-NEXT: v_or_b32_e32 v41, v1, v49
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_alignbit_b32 v1, v46, v17, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v55, v1, v39
+; SI-NEXT: v_alignbit_b32 v1, v44, v34, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v53, v1, v4
+; SI-NEXT: v_alignbit_b32 v1, v43, v35, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v4, v1, v6
+; SI-NEXT: v_alignbit_b32 v1, v42, v32, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v3, v1, v8
+; SI-NEXT: v_alignbit_b32 v1, v41, v36, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v37
+; SI-NEXT: v_or_b32_e32 v50, v1, v31
+; SI-NEXT: v_alignbit_b32 v1, v55, v45, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v5
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v53, v38, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v1, v4, v33, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v17
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v9
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v11
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v19
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v50, v2, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v1, v1, v13
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v21
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v1, v1, v15
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v1, v1, v17
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v23
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v1, v1, v34
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v1, v1, v35
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v1, v1, v32
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v1, v1, v36
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v1, v1, v45
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v1, v1, v38
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v1, v1, v33
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v63
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; kill: killed $vgpr0
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: ; implicit-def: $vgpr14
@@ -226473,458 +220966,438 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; kill: killed $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: .LBB106_2: ; %Flow
-; SI-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_xor_b64 exec, exec, s[4:5]
+; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB106_4
; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: v_add_i32_e32 v28, vcc, 3, v28
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v0, v28
+; SI-NEXT: s_mov_b32 s6, 0x30000
+; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v30
+; SI-NEXT: v_add_i32_e32 v28, vcc, 3, v29
+; SI-NEXT: v_add_i32_e32 v29, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v26
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v2, v2, v30
+; SI-NEXT: v_or_b32_e32 v8, v8, v28
+; SI-NEXT: v_or_b32_e32 v0, v33, v0
+; SI-NEXT: v_add_i32_e32 v28, vcc, 0x30000, v2
+; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v8
+; SI-NEXT: v_add_i32_e32 v8, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v27
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v6, v0
+; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v24
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v38, v0
+; SI-NEXT: v_add_i32_e32 v24, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v25
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v4, v0
+; SI-NEXT: v_add_i32_e32 v53, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v22
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v45, v0
+; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v23
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v39, v0
+; SI-NEXT: v_add_i32_e32 v55, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v20
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v36, v0
+; SI-NEXT: v_add_i32_e32 v20, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v21
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v49, v0
+; SI-NEXT: v_add_i32_e32 v41, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v18
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v32, v0
+; SI-NEXT: v_add_i32_e32 v18, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v19
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v48, v0
+; SI-NEXT: v_add_i32_e32 v42, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v35, v0
+; SI-NEXT: v_add_i32_e32 v16, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v37
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v31, v30
+; SI-NEXT: v_add_i32_e32 v50, vcc, s6, v30
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v50
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v17, v0
+; SI-NEXT: v_add_i32_e32 v43, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v14
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v34, v0
+; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v15, v0
+; SI-NEXT: v_add_i32_e32 v44, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v12, v0
+; SI-NEXT: v_add_i32_e32 v12, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v13, v0
+; SI-NEXT: v_add_i32_e32 v46, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v10, v0
+; SI-NEXT: v_add_i32_e32 v10, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v11, v0
+; SI-NEXT: v_add_i32_e32 v56, vcc, s6, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v56
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: s_mov_b32 s6, 0x30000
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, v31, v0
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v1, v32, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v2, v33, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_or_b32_e32 v3, v34, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_or_b32_e32 v4, v35, v4
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_or_b32_e32 v5, v36, v5
-; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: v_or_b32_e32 v6, v37, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
-; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_or_b32_e32 v7, v38, v7
-; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT: v_or_b32_e32 v8, v39, v8
-; SI-NEXT: v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_or_b32_e32 v9, v48, v9
-; SI-NEXT: v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT: v_or_b32_e32 v10, v49, v10
-; SI-NEXT: v_add_i32_e32 v9, vcc, s6, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_or_b32_e32 v11, v50, v11
-; SI-NEXT: v_add_i32_e32 v10, vcc, s6, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT: v_or_b32_e32 v12, v51, v12
-; SI-NEXT: v_add_i32_e32 v11, vcc, s6, v11
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: v_or_b32_e32 v13, v52, v13
-; SI-NEXT: v_add_i32_e32 v12, vcc, s6, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v11
-; SI-NEXT: v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT: v_or_b32_e32 v14, v53, v14
-; SI-NEXT: v_add_i32_e32 v13, vcc, s6, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v12
-; SI-NEXT: v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT: v_or_b32_e32 v15, v54, v15
-; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v13
-; SI-NEXT: v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT: v_or_b32_e32 v16, v55, v16
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
-; SI-NEXT: v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT: v_or_b32_e32 v17, v40, v17
-; SI-NEXT: v_add_i32_e32 v16, vcc, s6, v16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
-; SI-NEXT: v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT: v_or_b32_e32 v18, v41, v18
-; SI-NEXT: v_add_i32_e32 v17, vcc, s6, v17
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v16
-; SI-NEXT: v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: v_or_b32_e32 v19, v42, v19
-; SI-NEXT: v_add_i32_e32 v18, vcc, s6, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v17
-; SI-NEXT: v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT: v_or_b32_e32 v20, v43, v20
-; SI-NEXT: v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT: v_or_b32_e32 v0, v11, v0
+; SI-NEXT: v_add_i32_e32 v11, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v9
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v9, v0
+; SI-NEXT: v_add_i32_e32 v57, vcc, s6, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v57
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v9, v0
+; SI-NEXT: v_add_i32_e32 v9, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v7, v0
+; SI-NEXT: v_add_i32_e32 v59, vcc, s6, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v59
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v7, v0
+; SI-NEXT: v_add_i32_e32 v7, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v5, v0
+; SI-NEXT: v_add_i32_e32 v62, vcc, s6, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v62
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v5, v0
+; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v3, v0
+; SI-NEXT: v_add_i32_e32 v47, vcc, s6, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v47
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v3, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v61, vcc, s6, v0
+; SI-NEXT: v_alignbit_b32 v0, v61, v3, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
-; SI-NEXT: v_add_i32_e32 v23, vcc, 3, v23
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT: v_or_b32_e32 v21, v44, v21
-; SI-NEXT: v_add_i32_e32 v20, vcc, s6, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v19
-; SI-NEXT: v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT: v_or_b32_e32 v22, v45, v22
-; SI-NEXT: v_add_i32_e32 v21, vcc, s6, v21
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v47, v5, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT: v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT: v_or_b32_e32 v23, v46, v23
-; SI-NEXT: v_add_i32_e32 v22, vcc, s6, v22
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v21
-; SI-NEXT: v_add_i32_e32 v26, vcc, 3, v26
-; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; SI-NEXT: v_or_b32_e32 v24, v47, v24
-; SI-NEXT: v_add_i32_e32 v23, vcc, s6, v23
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_add_i32_e32 v27, vcc, 3, v27
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT: v_or_b32_e32 v25, v56, v25
-; SI-NEXT: v_add_i32_e32 v24, vcc, s6, v24
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v62, v7, 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v59, v9, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v57, v11, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v56, v10, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v46, v12, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v44, v14, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v43, v16, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v42, v18, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v41, v20, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v55, v4, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v53, v24, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v6, v8, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v2, v29, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_alignbit_b32 v0, v50, v28, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v23
-; SI-NEXT: v_add_i32_e32 v30, vcc, 3, v30
-; SI-NEXT: v_add_i32_e32 v29, vcc, 3, v29
-; SI-NEXT: v_add_i32_e32 v28, vcc, 3, v28
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; SI-NEXT: v_or_b32_e32 v26, v57, v26
-; SI-NEXT: v_add_i32_e32 v25, vcc, s6, v25
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; SI-NEXT: v_or_b32_e32 v27, v58, v27
-; SI-NEXT: v_add_i32_e32 v26, vcc, s6, v26
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v25
-; SI-NEXT: v_or_b32_e32 v30, v61, v30
-; SI-NEXT: v_or_b32_e32 v60, v60, v29
-; SI-NEXT: v_or_b32_e32 v59, v59, v28
-; SI-NEXT: v_add_i32_e32 v27, vcc, s6, v27
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v46
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v26
-; SI-NEXT: v_add_i32_e32 v63, vcc, 3, v63
-; SI-NEXT: v_add_i32_e32 v29, vcc, s6, v30
-; SI-NEXT: v_add_i32_e32 v30, vcc, s6, v60
-; SI-NEXT: v_add_i32_e32 v60, vcc, s6, v59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v44
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v27
-; SI-NEXT: v_and_b32_e32 v63, 0xffff, v63
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v43
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v60
-; SI-NEXT: v_or_b32_e32 v62, v62, v63
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v42
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
-; SI-NEXT: v_add_i32_e32 v28, vcc, 0x30000, v62
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v41
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v29
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v2
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v52, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v40, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v19
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v21
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v23
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v58, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v29
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v62, 0xffff0000, v28
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v55
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v53
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v61
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: .LBB106_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14) expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v31
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v32
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v34
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v36
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v38
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v48
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v50
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v52
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v54
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v58
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v63
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v60
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v40
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v54
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v51
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v52
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v40
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_alignbit_b32 v18, v18, v19, 16
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v42
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v43
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_alignbit_b32 v20, v20, v21, 16
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v44
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v21, v21, v22, 16
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_alignbit_b32 v22, v22, v23, 16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v46
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_alignbit_b32 v23, v23, v24, 16
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v47
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_alignbit_b32 v24, v24, v25, 16
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v56
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_alignbit_b32 v25, v25, v26, 16
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_alignbit_b32 v26, v26, v27, 16
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v58
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v27, v27, v28, 16
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v59
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_alignbit_b32 v28, v28, v29, 16
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v60
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v29, v29, v30, 16
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_alignbit_b32 v30, v30, v31, 16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v62
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v47
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v62
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v59
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v8, v8, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v57
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v56
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v46
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v44
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v16, v16, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v43
+; SI-NEXT: v_or_b32_e32 v17, v17, v18
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: v_or_b32_e32 v18, v18, v19
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v42
+; SI-NEXT: v_or_b32_e32 v19, v19, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_or_b32_e32 v20, v20, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v41
+; SI-NEXT: v_or_b32_e32 v21, v21, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@@ -226941,11 +221414,39 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v31, v31, v32, 16
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT: v_or_b32_e32 v22, v22, v23
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v55
+; SI-NEXT: v_or_b32_e32 v23, v23, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v24, v24, v25
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v53
+; SI-NEXT: v_or_b32_e32 v25, v25, v26
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
+; SI-NEXT: v_or_b32_e32 v26, v26, v27
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v27, v27, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT: v_or_b32_e32 v28, v28, v29
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_or_b32_e32 v30, v30, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v50
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v64i16_to_v64bf16:
@@ -227186,745 +221687,705 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_writelane_b32 v33, s34, 0
-; SI-NEXT: v_writelane_b32 v33, s35, 1
-; SI-NEXT: v_writelane_b32 v33, s36, 2
-; SI-NEXT: v_writelane_b32 v33, s37, 3
-; SI-NEXT: v_writelane_b32 v33, s38, 4
-; SI-NEXT: v_writelane_b32 v33, s39, 5
-; SI-NEXT: v_writelane_b32 v33, s48, 6
-; SI-NEXT: v_writelane_b32 v33, s49, 7
-; SI-NEXT: v_writelane_b32 v33, s50, 8
-; SI-NEXT: v_writelane_b32 v33, s51, 9
-; SI-NEXT: v_writelane_b32 v33, s52, 10
-; SI-NEXT: v_writelane_b32 v33, s53, 11
-; SI-NEXT: v_writelane_b32 v33, s54, 12
-; SI-NEXT: v_writelane_b32 v33, s55, 13
-; SI-NEXT: v_writelane_b32 v33, s64, 14
-; SI-NEXT: v_writelane_b32 v33, s65, 15
-; SI-NEXT: v_writelane_b32 v33, s66, 16
-; SI-NEXT: v_writelane_b32 v33, s67, 17
-; SI-NEXT: v_writelane_b32 v33, s68, 18
-; SI-NEXT: v_writelane_b32 v33, s69, 19
-; SI-NEXT: v_writelane_b32 v33, s70, 20
-; SI-NEXT: v_writelane_b32 v33, s71, 21
-; SI-NEXT: v_writelane_b32 v33, s80, 22
-; SI-NEXT: v_writelane_b32 v33, s81, 23
-; SI-NEXT: v_writelane_b32 v33, s82, 24
-; SI-NEXT: v_writelane_b32 v33, s83, 25
-; SI-NEXT: v_writelane_b32 v33, s84, 26
-; SI-NEXT: v_writelane_b32 v33, s85, 27
-; SI-NEXT: v_writelane_b32 v33, s86, 28
-; SI-NEXT: v_writelane_b32 v33, s87, 29
-; SI-NEXT: v_writelane_b32 v33, s96, 30
-; SI-NEXT: v_writelane_b32 v33, s97, 31
-; SI-NEXT: v_writelane_b32 v33, s98, 32
-; SI-NEXT: v_writelane_b32 v33, s99, 33
-; SI-NEXT: v_writelane_b32 v33, s30, 34
-; SI-NEXT: v_writelane_b32 v33, s31, 35
-; SI-NEXT: s_lshr_b32 s4, s16, 16
-; SI-NEXT: ; implicit-def: $vgpr34 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 s46, s17, 16
+; SI-NEXT: v_writelane_b32 v32, s34, 0
+; SI-NEXT: v_writelane_b32 v32, s35, 1
+; SI-NEXT: v_writelane_b32 v32, s36, 2
+; SI-NEXT: v_writelane_b32 v32, s37, 3
+; SI-NEXT: v_writelane_b32 v32, s38, 4
+; SI-NEXT: v_writelane_b32 v32, s39, 5
+; SI-NEXT: v_writelane_b32 v32, s48, 6
+; SI-NEXT: v_writelane_b32 v32, s49, 7
+; SI-NEXT: v_writelane_b32 v32, s50, 8
+; SI-NEXT: v_writelane_b32 v32, s51, 9
+; SI-NEXT: v_writelane_b32 v32, s52, 10
+; SI-NEXT: v_writelane_b32 v32, s53, 11
+; SI-NEXT: v_writelane_b32 v32, s54, 12
+; SI-NEXT: v_writelane_b32 v32, s55, 13
+; SI-NEXT: v_writelane_b32 v32, s64, 14
+; SI-NEXT: v_writelane_b32 v32, s65, 15
+; SI-NEXT: v_writelane_b32 v32, s66, 16
+; SI-NEXT: v_writelane_b32 v32, s67, 17
+; SI-NEXT: v_writelane_b32 v32, s68, 18
+; SI-NEXT: v_writelane_b32 v32, s69, 19
+; SI-NEXT: v_writelane_b32 v32, s70, 20
+; SI-NEXT: v_writelane_b32 v32, s71, 21
+; SI-NEXT: v_writelane_b32 v32, s80, 22
+; SI-NEXT: v_writelane_b32 v32, s81, 23
+; SI-NEXT: v_writelane_b32 v32, s82, 24
+; SI-NEXT: v_writelane_b32 v32, s83, 25
+; SI-NEXT: v_writelane_b32 v32, s84, 26
+; SI-NEXT: v_writelane_b32 v32, s85, 27
+; SI-NEXT: v_writelane_b32 v32, s86, 28
+; SI-NEXT: v_writelane_b32 v32, s87, 29
+; SI-NEXT: v_writelane_b32 v32, s96, 30
+; SI-NEXT: v_writelane_b32 v32, s97, 31
+; SI-NEXT: v_writelane_b32 v32, s98, 32
+; SI-NEXT: v_writelane_b32 v32, s99, 33
+; SI-NEXT: v_writelane_b32 v32, s30, 34
+; SI-NEXT: v_writelane_b32 v32, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s68, v0
+; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: s_lshr_b32 s14, s68, 16
+; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
+; SI-NEXT: v_readfirstlane_b32 s71, v4
+; SI-NEXT: s_lshr_b32 s12, s69, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v34, s4, 0
-; SI-NEXT: s_lshr_b32 s47, s18, 16
-; SI-NEXT: v_writelane_b32 v34, s46, 1
-; SI-NEXT: s_lshr_b32 s57, s19, 16
-; SI-NEXT: v_writelane_b32 v34, s47, 2
-; SI-NEXT: s_lshr_b32 s59, s20, 16
-; SI-NEXT: v_writelane_b32 v34, s57, 3
-; SI-NEXT: s_lshr_b32 s61, s21, 16
-; SI-NEXT: v_writelane_b32 v34, s59, 4
-; SI-NEXT: s_lshr_b32 s63, s22, 16
-; SI-NEXT: v_writelane_b32 v34, s61, 5
-; SI-NEXT: s_lshr_b32 s73, s23, 16
-; SI-NEXT: v_writelane_b32 v34, s63, 6
-; SI-NEXT: s_lshr_b32 s75, s24, 16
-; SI-NEXT: v_writelane_b32 v34, s73, 7
-; SI-NEXT: s_lshr_b32 s76, s25, 16
-; SI-NEXT: v_writelane_b32 v34, s75, 8
-; SI-NEXT: s_lshr_b32 s77, s26, 16
-; SI-NEXT: v_writelane_b32 v34, s76, 9
-; SI-NEXT: s_lshr_b32 s78, s27, 16
-; SI-NEXT: v_writelane_b32 v34, s77, 10
-; SI-NEXT: s_lshr_b32 s79, s28, 16
-; SI-NEXT: v_writelane_b32 v34, s78, 11
-; SI-NEXT: s_lshr_b32 s88, s29, 16
-; SI-NEXT: v_readfirstlane_b32 s54, v0
-; SI-NEXT: v_writelane_b32 v34, s79, 12
-; SI-NEXT: v_readfirstlane_b32 s64, v1
-; SI-NEXT: s_lshr_b32 s89, s54, 16
-; SI-NEXT: v_writelane_b32 v34, s88, 13
-; SI-NEXT: v_readfirstlane_b32 s66, v2
-; SI-NEXT: s_lshr_b32 s90, s64, 16
-; SI-NEXT: v_writelane_b32 v34, s89, 14
-; SI-NEXT: v_readfirstlane_b32 s68, v3
-; SI-NEXT: s_lshr_b32 s91, s66, 16
-; SI-NEXT: v_writelane_b32 v34, s90, 15
-; SI-NEXT: v_readfirstlane_b32 s70, v4
-; SI-NEXT: s_lshr_b32 s92, s68, 16
-; SI-NEXT: v_writelane_b32 v34, s91, 16
-; SI-NEXT: v_readfirstlane_b32 s80, v5
-; SI-NEXT: s_lshr_b32 s93, s70, 16
-; SI-NEXT: v_writelane_b32 v34, s92, 17
-; SI-NEXT: v_readfirstlane_b32 s82, v6
-; SI-NEXT: s_lshr_b32 s94, s80, 16
-; SI-NEXT: v_writelane_b32 v34, s93, 18
-; SI-NEXT: v_readfirstlane_b32 s84, v7
-; SI-NEXT: s_lshr_b32 s95, s82, 16
-; SI-NEXT: v_writelane_b32 v34, s94, 19
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: v_readfirstlane_b32 s40, v16
-; SI-NEXT: v_readfirstlane_b32 s14, v15
-; SI-NEXT: v_readfirstlane_b32 s12, v14
-; SI-NEXT: v_readfirstlane_b32 s10, v13
-; SI-NEXT: v_readfirstlane_b32 s8, v12
-; SI-NEXT: v_readfirstlane_b32 s6, v11
-; SI-NEXT: v_readfirstlane_b32 s98, v10
-; SI-NEXT: v_readfirstlane_b32 s96, v9
-; SI-NEXT: v_readfirstlane_b32 s86, v8
-; SI-NEXT: s_lshr_b32 vcc_lo, s84, 16
-; SI-NEXT: v_writelane_b32 v34, s95, 20
-; SI-NEXT: s_lshr_b32 s41, s42, 16
-; SI-NEXT: s_lshr_b32 s15, s40, 16
-; SI-NEXT: s_lshr_b32 s13, s14, 16
-; SI-NEXT: s_lshr_b32 s11, s12, 16
-; SI-NEXT: s_lshr_b32 s9, s10, 16
-; SI-NEXT: s_lshr_b32 s7, s8, 16
-; SI-NEXT: s_lshr_b32 s99, s6, 16
-; SI-NEXT: s_lshr_b32 s97, s98, 16
-; SI-NEXT: s_lshr_b32 s30, s96, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, s86, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v18
-; SI-NEXT: v_writelane_b32 v34, vcc_lo, 21
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_writelane_b32 v34, vcc_hi, 22
-; SI-NEXT: v_writelane_b32 v34, s30, 23
+; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_readfirstlane_b32 s80, v6
+; SI-NEXT: s_lshr_b32 s10, s71, 16
+; SI-NEXT: v_writelane_b32 v33, s12, 1
+; SI-NEXT: v_readfirstlane_b32 s82, v8
+; SI-NEXT: s_lshr_b32 s8, s80, 16
+; SI-NEXT: v_writelane_b32 v33, s10, 2
+; SI-NEXT: v_readfirstlane_b32 s84, v10
+; SI-NEXT: s_lshr_b32 s6, s82, 16
+; SI-NEXT: v_writelane_b32 v33, s8, 3
+; SI-NEXT: s_lshr_b32 s4, s84, 16
+; SI-NEXT: v_writelane_b32 v33, s6, 4
+; SI-NEXT: v_readfirstlane_b32 s15, v1
+; SI-NEXT: v_writelane_b32 v33, s4, 5
+; SI-NEXT: v_readfirstlane_b32 s13, v3
+; SI-NEXT: v_writelane_b32 v33, s15, 6
+; SI-NEXT: v_readfirstlane_b32 s11, v5
+; SI-NEXT: v_writelane_b32 v33, s13, 7
+; SI-NEXT: v_readfirstlane_b32 s9, v7
+; SI-NEXT: v_writelane_b32 v33, s11, 8
+; SI-NEXT: v_readfirstlane_b32 s7, v9
+; SI-NEXT: v_writelane_b32 v33, s9, 9
+; SI-NEXT: v_readfirstlane_b32 s5, v11
+; SI-NEXT: v_writelane_b32 v33, s7, 10
+; SI-NEXT: v_readfirstlane_b32 s75, v13
+; SI-NEXT: v_readfirstlane_b32 s86, v12
+; SI-NEXT: v_writelane_b32 v33, s5, 11
+; SI-NEXT: s_lshr_b32 s74, s86, 16
+; SI-NEXT: v_writelane_b32 v33, s75, 12
+; SI-NEXT: v_readfirstlane_b32 s72, v15
+; SI-NEXT: v_readfirstlane_b32 s96, v14
+; SI-NEXT: v_writelane_b32 v33, s74, 13
+; SI-NEXT: s_lshr_b32 s73, s96, 16
+; SI-NEXT: v_writelane_b32 v33, s72, 14
+; SI-NEXT: v_readfirstlane_b32 s62, v17
+; SI-NEXT: v_readfirstlane_b32 s98, v16
+; SI-NEXT: v_writelane_b32 v33, s73, 15
+; SI-NEXT: s_mov_b32 s76, s16
+; SI-NEXT: s_lshr_b32 vcc_hi, s29, 16
+; SI-NEXT: s_lshr_b32 s99, s28, 16
+; SI-NEXT: s_lshr_b32 s95, s27, 16
+; SI-NEXT: s_lshr_b32 s97, s26, 16
+; SI-NEXT: s_lshr_b32 s93, s25, 16
+; SI-NEXT: s_lshr_b32 s87, s24, 16
+; SI-NEXT: s_lshr_b32 s91, s23, 16
+; SI-NEXT: s_lshr_b32 s85, s22, 16
+; SI-NEXT: s_lshr_b32 s89, s21, 16
+; SI-NEXT: s_lshr_b32 s83, s20, 16
+; SI-NEXT: s_lshr_b32 s79, s19, 16
+; SI-NEXT: s_lshr_b32 s81, s18, 16
+; SI-NEXT: s_lshr_b32 s77, s17, 16
+; SI-NEXT: s_lshr_b32 s70, s16, 16
+; SI-NEXT: s_lshr_b32 s65, s62, 16
+; SI-NEXT: s_lshr_b32 s63, s98, 16
+; SI-NEXT: s_lshr_b32 s55, s72, 16
+; SI-NEXT: s_lshr_b32 s53, s75, 16
+; SI-NEXT: s_lshr_b32 s51, s5, 16
+; SI-NEXT: s_lshr_b32 s49, s7, 16
+; SI-NEXT: s_lshr_b32 s39, s9, 16
+; SI-NEXT: s_lshr_b32 s37, s11, 16
+; SI-NEXT: s_lshr_b32 s35, s13, 16
+; SI-NEXT: s_lshr_b32 s31, s15, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v18
+; SI-NEXT: v_writelane_b32 v33, s62, 16
+; SI-NEXT: s_cmp_lg_u32 s16, 0
+; SI-NEXT: v_writelane_b32 v33, s17, 17
+; SI-NEXT: v_writelane_b32 v33, s76, 18
; SI-NEXT: s_cbranch_scc0 .LBB107_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s44, s4, 16
-; SI-NEXT: s_lshl_b32 s4, s24, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 26
-; SI-NEXT: s_lshl_b32 s4, s25, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 27
-; SI-NEXT: s_lshl_b32 s4, s26, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 28
-; SI-NEXT: s_lshl_b32 s4, s27, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 29
-; SI-NEXT: s_lshl_b32 s4, s28, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 25
-; SI-NEXT: s_lshl_b32 s4, s79, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 30
-; SI-NEXT: s_lshl_b32 s4, s29, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 24
-; SI-NEXT: s_lshl_b32 s4, s88, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 31
-; SI-NEXT: s_lshl_b32 s4, s89, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 32
-; SI-NEXT: s_lshl_b32 s4, s90, 16
-; SI-NEXT: s_lshl_b32 s43, s16, 16
-; SI-NEXT: s_lshl_b32 s45, s17, 16
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s56, s18, 16
-; SI-NEXT: s_lshl_b32 s47, s47, 16
-; SI-NEXT: s_lshl_b32 s58, s19, 16
-; SI-NEXT: s_lshl_b32 s57, s57, 16
-; SI-NEXT: s_lshl_b32 s60, s20, 16
-; SI-NEXT: s_lshl_b32 s59, s59, 16
-; SI-NEXT: s_lshl_b32 s62, s21, 16
-; SI-NEXT: s_lshl_b32 s61, s61, 16
-; SI-NEXT: s_lshl_b32 s72, s22, 16
-; SI-NEXT: s_lshl_b32 s63, s63, 16
-; SI-NEXT: s_lshl_b32 s74, s23, 16
-; SI-NEXT: s_lshl_b32 s73, s73, 16
-; SI-NEXT: s_lshl_b32 s75, s75, 16
-; SI-NEXT: s_lshl_b32 s76, s76, 16
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_lshl_b32 s83, s54, 16
-; SI-NEXT: s_lshl_b32 s71, s64, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 33
-; SI-NEXT: s_lshl_b32 s81, s66, 16
-; SI-NEXT: s_lshl_b32 s87, s91, 16
-; SI-NEXT: s_lshl_b32 s55, s68, 16
-; SI-NEXT: s_lshl_b32 s85, s92, 16
-; SI-NEXT: s_lshl_b32 s52, s70, 16
-; SI-NEXT: s_lshl_b32 s67, s93, 16
-; SI-NEXT: s_lshl_b32 s50, s80, 16
-; SI-NEXT: s_lshl_b32 s69, s94, 16
-; SI-NEXT: s_lshl_b32 s48, s82, 16
-; SI-NEXT: s_lshl_b32 s65, s95, 16
-; SI-NEXT: s_lshl_b32 s38, s84, 16
-; SI-NEXT: s_lshl_b32 s53, vcc_lo, 16
-; SI-NEXT: s_lshl_b32 s36, s86, 16
-; SI-NEXT: s_lshl_b32 s51, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s34, s96, 16
-; SI-NEXT: s_lshl_b32 s49, s30, 16
-; SI-NEXT: s_lshl_b32 s30, s98, 16
-; SI-NEXT: s_lshl_b32 s39, s97, 16
-; SI-NEXT: s_lshl_b32 vcc_lo, s6, 16
-; SI-NEXT: s_lshl_b32 s37, s99, 16
-; SI-NEXT: s_lshl_b32 s94, s8, 16
-; SI-NEXT: s_lshl_b32 s35, s7, 16
-; SI-NEXT: s_lshl_b32 s92, s10, 16
-; SI-NEXT: s_lshl_b32 s31, s9, 16
-; SI-NEXT: s_lshl_b32 s95, s12, 16
-; SI-NEXT: s_lshl_b32 vcc_hi, s11, 16
-; SI-NEXT: s_lshl_b32 s89, s14, 16
-; SI-NEXT: s_lshl_b32 s93, s13, 16
-; SI-NEXT: s_lshl_b32 s90, s40, 16
-; SI-NEXT: s_lshl_b32 s91, s15, 16
-; SI-NEXT: s_lshl_b32 s79, s42, 16
-; SI-NEXT: s_lshl_b32 s88, s41, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s16, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s77, 16
+; SI-NEXT: s_or_b32 s61, s16, s17
+; SI-NEXT: s_and_b32 s16, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s79, 16
+; SI-NEXT: s_or_b32 s59, s16, s17
+; SI-NEXT: s_and_b32 s16, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s89, 16
+; SI-NEXT: s_or_b32 s57, s16, s17
+; SI-NEXT: s_and_b32 s16, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s91, 16
+; SI-NEXT: s_or_b32 s47, s16, s17
+; SI-NEXT: s_and_b32 s16, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s93, 16
+; SI-NEXT: s_or_b32 s45, s16, s17
+; SI-NEXT: s_and_b32 s16, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s95, 16
+; SI-NEXT: s_or_b32 s43, s16, s17
+; SI-NEXT: s_and_b32 s16, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s17, vcc_hi, 16
+; SI-NEXT: s_or_b32 s41, s16, s17
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s31, 16
+; SI-NEXT: s_or_b32 s15, s15, s16
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s35, 16
+; SI-NEXT: s_or_b32 s13, s13, s16
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s37, 16
+; SI-NEXT: s_or_b32 s11, s11, s16
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s39, 16
+; SI-NEXT: s_or_b32 s9, s9, s16
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s49, 16
+; SI-NEXT: s_or_b32 s7, s7, s16
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s51, 16
+; SI-NEXT: s_or_b32 s5, s5, s16
+; SI-NEXT: s_and_b32 s16, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s53, 16
+; SI-NEXT: s_mov_b32 s66, s53
+; SI-NEXT: s_or_b32 s53, s16, s17
+; SI-NEXT: s_and_b32 s16, s72, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s55, 16
+; SI-NEXT: s_mov_b32 s67, s55
+; SI-NEXT: s_or_b32 s55, s16, s17
+; SI-NEXT: s_and_b32 s16, s62, 0xffff
+; SI-NEXT: s_lshl_b32 s62, s65, 16
+; SI-NEXT: s_lshl_b32 s60, s70, 16
+; SI-NEXT: s_mov_b32 s17, s65
+; SI-NEXT: s_or_b32 s65, s16, s62
+; SI-NEXT: s_and_b32 s62, s76, 0xffff
+; SI-NEXT: s_lshl_b32 s58, s81, 16
+; SI-NEXT: s_or_b32 s62, s62, s60
+; SI-NEXT: s_mov_b32 s72, s77
+; SI-NEXT: s_lshr_b64 s[76:77], s[60:61], 16
+; SI-NEXT: s_and_b32 s60, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s83, 16
+; SI-NEXT: s_lshl_b32 s52, s74, 16
+; SI-NEXT: s_or_b32 s74, s60, s58
+; SI-NEXT: s_mov_b32 s60, s79
+; SI-NEXT: s_lshr_b64 s[78:79], s[58:59], 16
+; SI-NEXT: s_and_b32 s58, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s85, 16
+; SI-NEXT: s_mov_b32 s77, s72
+; SI-NEXT: s_or_b32 s72, s58, s56
+; SI-NEXT: s_mov_b32 s58, s89
+; SI-NEXT: s_lshr_b64 s[88:89], s[56:57], 16
+; SI-NEXT: s_and_b32 s56, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s44, s87, 16
+; SI-NEXT: s_mov_b32 s79, s60
+; SI-NEXT: s_or_b32 s60, s56, s46
+; SI-NEXT: s_mov_b32 s56, s91
+; SI-NEXT: s_lshr_b64 s[90:91], s[46:47], 16
+; SI-NEXT: s_and_b32 s46, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s42, s97, 16
+; SI-NEXT: s_mov_b32 s89, s58
+; SI-NEXT: s_or_b32 s58, s46, s44
+; SI-NEXT: s_mov_b32 s46, s93
+; SI-NEXT: s_lshr_b64 s[92:93], s[44:45], 16
+; SI-NEXT: s_and_b32 s44, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s99, 16
+; SI-NEXT: s_mov_b32 s91, s56
+; SI-NEXT: s_or_b32 s56, s44, s42
+; SI-NEXT: s_mov_b32 s44, s95
+; SI-NEXT: s_lshr_b64 s[94:95], s[42:43], 16
+; SI-NEXT: s_and_b32 s42, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_mov_b32 s93, s46
+; SI-NEXT: s_or_b32 s46, s42, s40
+; SI-NEXT: s_mov_b32 s42, vcc_hi
+; SI-NEXT: s_lshr_b64 vcc, s[40:41], 16
+; SI-NEXT: s_and_b32 s40, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_mov_b32 s95, s44
+; SI-NEXT: s_or_b32 s44, s40, s14
+; SI-NEXT: s_mov_b32 s40, s31
+; SI-NEXT: s_lshr_b64 s[30:31], s[14:15], 16
+; SI-NEXT: s_and_b32 s14, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_mov_b32 vcc_hi, s42
+; SI-NEXT: s_or_b32 s42, s14, s12
+; SI-NEXT: s_mov_b32 s14, s35
+; SI-NEXT: s_lshr_b64 s[34:35], s[12:13], 16
+; SI-NEXT: s_and_b32 s12, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_mov_b32 s31, s40
+; SI-NEXT: s_or_b32 s40, s12, s10
+; SI-NEXT: s_mov_b32 s12, s37
+; SI-NEXT: s_lshr_b64 s[36:37], s[10:11], 16
+; SI-NEXT: s_and_b32 s10, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_mov_b32 s35, s14
+; SI-NEXT: s_or_b32 s14, s10, s8
+; SI-NEXT: s_mov_b32 s10, s39
+; SI-NEXT: s_lshr_b64 s[38:39], s[8:9], 16
+; SI-NEXT: s_and_b32 s8, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_mov_b32 s37, s12
+; SI-NEXT: s_or_b32 s12, s8, s6
+; SI-NEXT: s_mov_b32 s8, s49
+; SI-NEXT: s_lshr_b64 s[48:49], s[6:7], 16
+; SI-NEXT: s_and_b32 s6, s84, 0xffff
+; SI-NEXT: s_mov_b32 s39, s10
+; SI-NEXT: s_or_b32 s10, s6, s4
+; SI-NEXT: s_mov_b32 s6, s51
+; SI-NEXT: s_lshr_b64 s[50:51], s[4:5], 16
+; SI-NEXT: s_and_b32 s4, s86, 0xffff
+; SI-NEXT: s_lshl_b32 s54, s73, 16
+; SI-NEXT: s_mov_b32 s49, s8
+; SI-NEXT: s_or_b32 s8, s4, s52
+; SI-NEXT: s_and_b32 s4, s96, 0xffff
+; SI-NEXT: s_lshl_b32 s64, s63, 16
+; SI-NEXT: s_mov_b32 s51, s6
+; SI-NEXT: s_or_b32 s6, s4, s54
+; SI-NEXT: s_and_b32 s4, s98, 0xffff
+; SI-NEXT: s_mov_b32 s16, s63
+; SI-NEXT: s_mov_b32 s63, s61
+; SI-NEXT: s_mov_b32 s75, s59
+; SI-NEXT: s_mov_b32 s73, s57
+; SI-NEXT: s_mov_b32 s61, s47
+; SI-NEXT: s_mov_b32 s59, s45
+; SI-NEXT: s_mov_b32 s57, s43
+; SI-NEXT: s_mov_b32 s47, s41
+; SI-NEXT: s_mov_b32 s45, s15
+; SI-NEXT: s_mov_b32 s43, s13
+; SI-NEXT: s_mov_b32 s41, s11
+; SI-NEXT: s_mov_b32 s15, s9
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: s_mov_b32 s11, s5
+; SI-NEXT: s_mov_b32 s9, s53
+; SI-NEXT: s_lshr_b64 s[52:53], s[52:53], 16
+; SI-NEXT: s_mov_b32 s7, s55
+; SI-NEXT: s_lshr_b64 s[54:55], s[54:55], 16
+; SI-NEXT: s_or_b32 s4, s4, s64
+; SI-NEXT: s_mov_b32 s5, s65
+; SI-NEXT: s_lshr_b64 s[64:65], s[64:65], 16
+; SI-NEXT: s_mov_b32 s53, s66
+; SI-NEXT: s_mov_b32 s55, s67
+; SI-NEXT: s_mov_b32 s65, s17
+; SI-NEXT: s_mov_b64 s[66:67], 0
; SI-NEXT: s_branch .LBB107_3
; SI-NEXT: .LBB107_2:
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; kill: killed $sgpr79
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; kill: killed $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: s_mov_b32 s16, s63
+; SI-NEXT: s_mov_b64 s[66:67], -1
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; kill: killed $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; kill: killed $sgpr79
-; SI-NEXT: ; kill: killed $sgpr76
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; kill: killed $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; kill: killed $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; kill: killed $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; kill: killed $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; kill: killed $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr64
; SI-NEXT: .LBB107_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_mov_b32 s4, s79
-; SI-NEXT: s_mov_b32 s5, s90
-; SI-NEXT: s_mov_b32 s79, s89
-; SI-NEXT: s_mov_b32 s89, s95
-; SI-NEXT: s_mov_b32 s90, s92
-; SI-NEXT: s_mov_b32 s92, s94
-; SI-NEXT: s_mov_b32 s94, vcc_lo
-; SI-NEXT: s_mov_b32 s95, vcc_hi
-; SI-NEXT: s_mov_b32 vcc_lo, s30
-; SI-NEXT: s_mov_b32 vcc_hi, s31
-; SI-NEXT: s_mov_b32 s30, s34
-; SI-NEXT: s_mov_b32 s31, s35
-; SI-NEXT: s_mov_b32 s34, s36
-; SI-NEXT: s_mov_b32 s35, s37
-; SI-NEXT: s_mov_b32 s36, s38
-; SI-NEXT: s_mov_b32 s37, s39
-; SI-NEXT: s_mov_b32 s38, s48
-; SI-NEXT: s_mov_b32 s39, s49
-; SI-NEXT: s_mov_b32 s48, s50
-; SI-NEXT: s_mov_b32 s49, s51
-; SI-NEXT: s_mov_b32 s50, s52
-; SI-NEXT: s_mov_b32 s51, s53
-; SI-NEXT: s_mov_b32 s52, s55
-; SI-NEXT: s_mov_b32 s53, s65
-; SI-NEXT: s_mov_b32 s55, s81
-; SI-NEXT: s_mov_b32 s65, s69
-; SI-NEXT: s_mov_b32 s69, s85
-; SI-NEXT: v_readlane_b32 s81, v34, 24
-; SI-NEXT: v_readlane_b32 s85, v34, 25
+; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
+; SI-NEXT: s_cselect_b32 s17, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s17, 1
; SI-NEXT: s_cbranch_scc1 .LBB107_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s8, s8, 3
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: s_add_i32 s10, s10, 3
-; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s99, 16
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_add_i32 s98, s98, 3
-; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_add_i32 s8, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s98, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s97, 16
-; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_or_b32 s6, s10, s6
+; SI-NEXT: s_and_b32 s4, s98, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_readlane_b32 s5, v33, 16
+; SI-NEXT: s_add_i32 s5, s5, 3
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s65, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s96, s96, 3
-; SI-NEXT: v_readlane_b32 s12, v34, 23
-; SI-NEXT: s_add_i32 s42, s42, 3
-; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_add_i32 s10, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s96, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s4, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s41, 16
-; SI-NEXT: s_add_i32 s40, s40, 3
-; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_or_b32 s6, s12, s6
+; SI-NEXT: v_readlane_b32 s6, v33, 15
+; SI-NEXT: s_and_b32 s7, s96, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: v_readlane_b32 s7, v33, 14
+; SI-NEXT: s_add_i32 s7, s7, 3
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s55, 16
+; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_add_i32 s86, s86, 3
-; SI-NEXT: v_readlane_b32 s14, v34, 22
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_add_i32 s12, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_or_b32 s5, s15, s5
-; SI-NEXT: s_or_b32 s6, s14, s6
+; SI-NEXT: v_readlane_b32 s8, v33, 13
+; SI-NEXT: s_and_b32 s9, s86, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: v_readlane_b32 s9, v33, 12
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s53, 16
+; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_add_i32 s84, s84, 3
-; SI-NEXT: v_readlane_b32 s15, v34, 21
-; SI-NEXT: s_add_i32 s14, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_or_b32 s6, s15, s6
+; SI-NEXT: v_readlane_b32 s10, v33, 5
+; SI-NEXT: s_and_b32 s11, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: v_readlane_b32 s11, v33, 11
+; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s51, 16
+; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: s_add_i32 s82, s82, 3
-; SI-NEXT: v_readlane_b32 s40, v34, 20
-; SI-NEXT: s_add_i32 s15, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s82, 0xffff
-; SI-NEXT: s_lshl_b32 s40, s40, 16
-; SI-NEXT: s_or_b32 s6, s40, s6
+; SI-NEXT: v_readlane_b32 s12, v33, 4
+; SI-NEXT: s_and_b32 s13, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s12, s12, s13
+; SI-NEXT: v_readlane_b32 s13, v33, 10
+; SI-NEXT: s_add_i32 s13, s13, 3
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s49, 16
+; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: s_add_i32 s80, s80, 3
-; SI-NEXT: v_readlane_b32 s41, v34, 19
-; SI-NEXT: s_add_i32 s40, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s41, 16
-; SI-NEXT: s_or_b32 s6, s41, s6
-; SI-NEXT: s_add_i32 s70, s70, 3
-; SI-NEXT: v_readlane_b32 s42, v34, 18
-; SI-NEXT: s_add_i32 s41, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_or_b32 s6, s42, s6
+; SI-NEXT: v_readlane_b32 s14, v33, 3
+; SI-NEXT: s_and_b32 s15, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_or_b32 s14, s14, s15
+; SI-NEXT: v_readlane_b32 s15, v33, 9
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s39, 16
+; SI-NEXT: s_or_b32 s15, s16, s15
+; SI-NEXT: s_add_i32 s71, s71, 3
+; SI-NEXT: v_readlane_b32 s16, v33, 2
+; SI-NEXT: s_and_b32 s17, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_add_i32 s40, s16, 0x30000
+; SI-NEXT: v_readlane_b32 s16, v33, 8
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s37, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s41, s16, 0x30000
+; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: v_readlane_b32 s16, v33, 1
+; SI-NEXT: s_and_b32 s17, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_add_i32 s42, s16, 0x30000
+; SI-NEXT: v_readlane_b32 s16, v33, 7
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s35, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s43, s16, 0x30000
; SI-NEXT: s_add_i32 s68, s68, 3
-; SI-NEXT: v_readlane_b32 s43, v34, 17
-; SI-NEXT: s_add_i32 s42, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s43, 16
-; SI-NEXT: s_or_b32 s6, s43, s6
-; SI-NEXT: s_add_i32 s66, s66, 3
-; SI-NEXT: v_readlane_b32 s43, v34, 16
-; SI-NEXT: s_add_i32 s67, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s43, 16
-; SI-NEXT: s_or_b32 s6, s43, s6
-; SI-NEXT: s_add_i32 s64, s64, 3
-; SI-NEXT: v_readlane_b32 s43, v34, 15
-; SI-NEXT: s_add_i32 s65, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s64, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s43, 16
-; SI-NEXT: s_or_b32 s6, s43, s6
-; SI-NEXT: s_add_i32 s54, s54, 3
-; SI-NEXT: v_readlane_b32 s43, v34, 14
-; SI-NEXT: s_add_i32 s88, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s54, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s43, 16
-; SI-NEXT: s_or_b32 s6, s43, s6
-; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_add_i32 s90, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s29, 0xffff
-; SI-NEXT: v_readlane_b32 s29, v34, 13
-; SI-NEXT: s_lshl_b32 s29, s29, 16
-; SI-NEXT: s_or_b32 s6, s29, s6
+; SI-NEXT: v_readlane_b32 s16, v33, 0
+; SI-NEXT: s_and_b32 s17, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_add_i32 s44, s16, 0x30000
+; SI-NEXT: v_readlane_b32 s16, v33, 6
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s31, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_add_i32 s29, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s28, 0xffff
-; SI-NEXT: v_readlane_b32 s28, v34, 12
-; SI-NEXT: s_lshl_b32 s28, s28, 16
-; SI-NEXT: s_or_b32 s6, s28, s6
-; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_add_i32 s28, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s27, 0xffff
-; SI-NEXT: v_readlane_b32 s27, v34, 11
-; SI-NEXT: s_lshl_b32 s27, s27, 16
-; SI-NEXT: s_or_b32 s6, s27, s6
+; SI-NEXT: s_add_i32 s45, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s99, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s29, s29, 3
+; SI-NEXT: s_add_i32 s46, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s17, vcc_hi, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_add_i32 s27, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s26, 0xffff
-; SI-NEXT: v_readlane_b32 s26, v34, 10
-; SI-NEXT: s_lshl_b32 s26, s26, 16
-; SI-NEXT: s_or_b32 s6, s26, s6
-; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_add_i32 s26, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s25, 0xffff
-; SI-NEXT: v_readlane_b32 s25, v34, 9
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s6, s25, s6
+; SI-NEXT: s_add_i32 s47, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s97, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s27, s27, 3
+; SI-NEXT: s_add_i32 s56, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s95, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_add_i32 s25, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s24, 0xffff
-; SI-NEXT: v_readlane_b32 s24, v34, 8
-; SI-NEXT: s_lshl_b32 s24, s24, 16
-; SI-NEXT: s_or_b32 s6, s24, s6
-; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_add_i32 s24, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s23, 0xffff
-; SI-NEXT: v_readlane_b32 s23, v34, 7
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_or_b32 s6, s23, s6
+; SI-NEXT: s_add_i32 s57, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s87, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s25, s25, 3
+; SI-NEXT: s_add_i32 s58, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s93, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_add_i32 s23, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s22, 0xffff
-; SI-NEXT: v_readlane_b32 s22, v34, 6
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_or_b32 s6, s22, s6
-; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s22, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s21, 0xffff
-; SI-NEXT: v_readlane_b32 s21, v34, 5
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_or_b32 s6, s21, s6
+; SI-NEXT: s_add_i32 s59, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s85, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s23, s23, 3
+; SI-NEXT: s_add_i32 s60, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s91, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_add_i32 s21, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s20, 0xffff
-; SI-NEXT: v_readlane_b32 s20, v34, 4
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_or_b32 s6, s20, s6
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_add_i32 s20, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s19, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v34, 3
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_or_b32 s6, s19, s6
+; SI-NEXT: s_add_i32 s61, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s83, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s21, s21, 3
+; SI-NEXT: s_add_i32 s72, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s89, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_add_i32 s19, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v34, 2
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_or_b32 s6, s18, s6
-; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s18, s6, 0x30000
-; SI-NEXT: s_and_b32 s6, s17, 0xffff
-; SI-NEXT: v_readlane_b32 s17, v34, 1
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_or_b32 s6, s17, s6
-; SI-NEXT: s_add_i32 s17, s6, 0x30000
+; SI-NEXT: s_add_i32 s73, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s81, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s19, s19, 3
+; SI-NEXT: s_add_i32 s74, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s79, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s75, s16, 0x30000
+; SI-NEXT: v_readlane_b32 s16, v33, 18
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s6, v34, 0
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_or_b32 s6, s6, s16
-; SI-NEXT: s_add_i32 s6, s6, 0x30000
-; SI-NEXT: s_and_b32 s44, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s6, 16
-; SI-NEXT: s_lshl_b32 s6, s24, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 26
-; SI-NEXT: s_lshl_b32 s6, s25, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 27
-; SI-NEXT: s_lshl_b32 s6, s26, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 28
-; SI-NEXT: s_lshl_b32 s6, s27, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 29
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s6, 30
-; SI-NEXT: s_and_b32 s6, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s17, s70, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s62, s16, 0x30000
+; SI-NEXT: v_readlane_b32 s16, v33, 17
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s77, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: s_add_i32 s9, s9, 0x30000
+; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s6, 31
-; SI-NEXT: s_and_b32 s6, s90, 0xffff0000
-; SI-NEXT: s_and_b32 s46, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s17, 16
-; SI-NEXT: s_and_b32 s47, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s18, 16
-; SI-NEXT: s_and_b32 s57, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s19, 16
-; SI-NEXT: s_and_b32 s59, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s20, 16
-; SI-NEXT: s_and_b32 s61, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s21, 16
-; SI-NEXT: s_and_b32 s63, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s22, 16
-; SI-NEXT: s_and_b32 s73, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s23, 16
-; SI-NEXT: s_and_b32 s75, s24, 0xffff0000
-; SI-NEXT: s_and_b32 s76, s25, 0xffff0000
-; SI-NEXT: s_and_b32 s77, s26, 0xffff0000
-; SI-NEXT: s_and_b32 s78, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s28, 16
-; SI-NEXT: s_lshl_b32 s81, s29, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 32
-; SI-NEXT: s_lshl_b32 s83, s90, 16
-; SI-NEXT: s_and_b32 s6, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s88, 16
-; SI-NEXT: s_and_b32 s87, s65, 0xffff0000
-; SI-NEXT: s_lshl_b32 s55, s65, 16
-; SI-NEXT: s_and_b32 s69, s67, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s67, 16
-; SI-NEXT: s_and_b32 s67, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s42, 16
-; SI-NEXT: s_and_b32 s65, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s41, 16
-; SI-NEXT: s_and_b32 s53, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s40, 16
-; SI-NEXT: s_and_b32 s51, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s15, 16
-; SI-NEXT: s_and_b32 s49, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s14, 16
-; SI-NEXT: s_and_b32 s39, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s12, 16
-; SI-NEXT: s_and_b32 s37, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, s10, 16
-; SI-NEXT: s_and_b32 s35, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s8, 16
-; SI-NEXT: s_and_b32 s31, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s7, 16
-; SI-NEXT: s_and_b32 vcc_hi, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s9, 16
-; SI-NEXT: s_and_b32 s95, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s89, s11, 16
-; SI-NEXT: s_and_b32 s93, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s79, s13, 16
-; SI-NEXT: s_and_b32 s91, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_and_b32 s88, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 33
+; SI-NEXT: s_add_i32 s8, s8, 0x30000
+; SI-NEXT: s_add_i32 s9, s9, 0x30000
+; SI-NEXT: s_add_i32 s10, s10, 0x30000
+; SI-NEXT: s_add_i32 s11, s11, 0x30000
+; SI-NEXT: s_add_i32 s12, s12, 0x30000
+; SI-NEXT: s_add_i32 s13, s13, 0x30000
+; SI-NEXT: s_add_i32 s14, s14, 0x30000
+; SI-NEXT: s_add_i32 s15, s15, 0x30000
+; SI-NEXT: s_add_i32 s63, s16, 0x30000
+; SI-NEXT: s_lshr_b64 s[76:77], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[74:75], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[56:57], 16
+; SI-NEXT: s_lshr_b64 vcc, s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[50:51], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s77, s63, 16
+; SI-NEXT: s_lshr_b32 s79, s75, 16
+; SI-NEXT: s_lshr_b32 s89, s73, 16
+; SI-NEXT: s_lshr_b32 s91, s61, 16
+; SI-NEXT: s_lshr_b32 s93, s59, 16
+; SI-NEXT: s_lshr_b32 s95, s57, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s47, 16
+; SI-NEXT: s_lshr_b32 s31, s45, 16
+; SI-NEXT: s_lshr_b32 s35, s43, 16
+; SI-NEXT: s_lshr_b32 s37, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s15, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s51, s11, 16
+; SI-NEXT: s_lshr_b32 s53, s9, 16
+; SI-NEXT: s_lshr_b32 s55, s7, 16
+; SI-NEXT: s_lshr_b32 s65, s5, 16
; SI-NEXT: .LBB107_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s44
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s46
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s45
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s47
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s56
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s57
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s58
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s59
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s60
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s61
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s63
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s72
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s73
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s74
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s75
-; SI-NEXT: v_readlane_b32 s6, v34, 26
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s76
-; SI-NEXT: v_readlane_b32 s6, v34, 27
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s77
-; SI-NEXT: v_readlane_b32 s6, v34, 28
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s78
-; SI-NEXT: v_readlane_b32 s6, v34, 29
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 30
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s6
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s85
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 31
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s6
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s81
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 32
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s6
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s83
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 33
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s71
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s87
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s55
-; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s69
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s52
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s67
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s50
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s65
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s48
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s53
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s38
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s51
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s36
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s49
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s34
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s39
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s30
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s37
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, vcc_lo
-; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s35
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s94
-; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s31
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s92
-; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, vcc_hi
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s90
-; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s95
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s89
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s93
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s79
-; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s91
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s5
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s88
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
-; SI-NEXT: v_readlane_b32 s30, v33, 34
-; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
-; SI-NEXT: v_readlane_b32 s31, v33, 35
-; SI-NEXT: v_readlane_b32 s99, v33, 33
-; SI-NEXT: v_readlane_b32 s98, v33, 32
-; SI-NEXT: v_readlane_b32 s97, v33, 31
-; SI-NEXT: v_readlane_b32 s96, v33, 30
-; SI-NEXT: v_readlane_b32 s87, v33, 29
-; SI-NEXT: v_readlane_b32 s86, v33, 28
-; SI-NEXT: v_readlane_b32 s85, v33, 27
-; SI-NEXT: v_readlane_b32 s84, v33, 26
-; SI-NEXT: v_readlane_b32 s83, v33, 25
-; SI-NEXT: v_readlane_b32 s82, v33, 24
-; SI-NEXT: v_readlane_b32 s81, v33, 23
-; SI-NEXT: v_readlane_b32 s80, v33, 22
-; SI-NEXT: v_readlane_b32 s71, v33, 21
-; SI-NEXT: v_readlane_b32 s70, v33, 20
-; SI-NEXT: v_readlane_b32 s69, v33, 19
-; SI-NEXT: v_readlane_b32 s68, v33, 18
-; SI-NEXT: v_readlane_b32 s67, v33, 17
-; SI-NEXT: v_readlane_b32 s66, v33, 16
-; SI-NEXT: v_readlane_b32 s65, v33, 15
-; SI-NEXT: v_readlane_b32 s64, v33, 14
-; SI-NEXT: v_readlane_b32 s55, v33, 13
-; SI-NEXT: v_readlane_b32 s54, v33, 12
-; SI-NEXT: v_readlane_b32 s53, v33, 11
-; SI-NEXT: v_readlane_b32 s52, v33, 10
-; SI-NEXT: v_readlane_b32 s51, v33, 9
-; SI-NEXT: v_readlane_b32 s50, v33, 8
-; SI-NEXT: v_readlane_b32 s49, v33, 7
-; SI-NEXT: v_readlane_b32 s48, v33, 6
-; SI-NEXT: v_readlane_b32 s39, v33, 5
-; SI-NEXT: v_readlane_b32 s38, v33, 4
-; SI-NEXT: v_readlane_b32 s37, v33, 3
-; SI-NEXT: v_readlane_b32 s36, v33, 2
-; SI-NEXT: v_readlane_b32 s35, v33, 1
-; SI-NEXT: v_readlane_b32 s34, v33, 0
+; SI-NEXT: s_and_b32 s16, s62, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s76, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_and_b32 s17, s63, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s77, 16
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_and_b32 s18, s74, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s78, 16
+; SI-NEXT: s_or_b32 s18, s18, s19
+; SI-NEXT: s_and_b32 s19, s75, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s79, 16
+; SI-NEXT: s_or_b32 s19, s19, s20
+; SI-NEXT: s_and_b32 s20, s72, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s88, 16
+; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_and_b32 s21, s73, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s89, 16
+; SI-NEXT: s_or_b32 s21, s21, s22
+; SI-NEXT: s_and_b32 s22, s60, 0xffff
+; SI-NEXT: s_lshl_b32 s23, s90, 16
+; SI-NEXT: s_or_b32 s22, s22, s23
+; SI-NEXT: s_and_b32 s23, s61, 0xffff
+; SI-NEXT: s_lshl_b32 s24, s91, 16
+; SI-NEXT: s_or_b32 s23, s23, s24
+; SI-NEXT: s_and_b32 s24, s58, 0xffff
+; SI-NEXT: s_lshl_b32 s25, s92, 16
+; SI-NEXT: s_or_b32 s24, s24, s25
+; SI-NEXT: s_and_b32 s25, s59, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s93, 16
+; SI-NEXT: s_or_b32 s25, s25, s26
+; SI-NEXT: s_and_b32 s26, s56, 0xffff
+; SI-NEXT: s_lshl_b32 s27, s94, 16
+; SI-NEXT: s_or_b32 s26, s26, s27
+; SI-NEXT: s_and_b32 s27, s57, 0xffff
+; SI-NEXT: s_lshl_b32 s28, s95, 16
+; SI-NEXT: s_or_b32 s27, s27, s28
+; SI-NEXT: s_and_b32 s28, s46, 0xffff
+; SI-NEXT: s_lshl_b32 s29, vcc_lo, 16
+; SI-NEXT: s_or_b32 s28, s28, s29
+; SI-NEXT: s_and_b32 s29, s47, 0xffff
+; SI-NEXT: s_lshl_b32 s46, vcc_hi, 16
+; SI-NEXT: s_or_b32 s29, s29, s46
+; SI-NEXT: s_and_b32 s44, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s30, 16
+; SI-NEXT: s_or_b32 s44, s44, s46
+; SI-NEXT: s_and_b32 s45, s45, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s31, 16
+; SI-NEXT: s_or_b32 s45, s45, s46
+; SI-NEXT: s_and_b32 s42, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s34, 16
+; SI-NEXT: s_or_b32 s42, s42, s46
+; SI-NEXT: s_and_b32 s43, s43, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s35, 16
+; SI-NEXT: s_or_b32 s43, s43, s46
+; SI-NEXT: s_and_b32 s40, s40, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s36, 16
+; SI-NEXT: s_or_b32 s40, s40, s46
+; SI-NEXT: s_and_b32 s41, s41, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s37, 16
+; SI-NEXT: s_or_b32 s41, s41, s46
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s38, 16
+; SI-NEXT: s_or_b32 s14, s14, s46
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s39, 16
+; SI-NEXT: s_or_b32 s15, s15, s46
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s48, 16
+; SI-NEXT: s_or_b32 s12, s12, s46
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s49, 16
+; SI-NEXT: s_or_b32 s13, s13, s46
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s50, 16
+; SI-NEXT: s_or_b32 s10, s10, s46
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s51, 16
+; SI-NEXT: s_or_b32 s11, s11, s46
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s52, 16
+; SI-NEXT: s_or_b32 s8, s8, s46
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s53, 16
+; SI-NEXT: s_or_b32 s9, s9, s46
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s54, 16
+; SI-NEXT: s_or_b32 s6, s6, s46
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s55, 16
+; SI-NEXT: s_or_b32 s7, s7, s46
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s64, 16
+; SI-NEXT: s_or_b32 s4, s4, s46
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s65, 16
+; SI-NEXT: s_or_b32 s5, s5, s46
+; SI-NEXT: v_readlane_b32 s30, v32, 34
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v14, s44
+; SI-NEXT: v_mov_b32_e32 v15, s45
+; SI-NEXT: v_mov_b32_e32 v16, s42
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v18, s40
+; SI-NEXT: v_mov_b32_e32 v19, s41
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v21, s15
+; SI-NEXT: v_mov_b32_e32 v22, s12
+; SI-NEXT: v_mov_b32_e32 v23, s13
+; SI-NEXT: v_mov_b32_e32 v24, s10
+; SI-NEXT: v_mov_b32_e32 v25, s11
+; SI-NEXT: v_mov_b32_e32 v26, s8
+; SI-NEXT: v_mov_b32_e32 v27, s9
+; SI-NEXT: v_mov_b32_e32 v28, s6
+; SI-NEXT: v_mov_b32_e32 v29, s7
+; SI-NEXT: v_mov_b32_e32 v30, s4
+; SI-NEXT: v_mov_b32_e32 v31, s5
+; SI-NEXT: v_readlane_b32 s31, v32, 35
+; SI-NEXT: v_readlane_b32 s99, v32, 33
+; SI-NEXT: v_readlane_b32 s98, v32, 32
+; SI-NEXT: v_readlane_b32 s97, v32, 31
+; SI-NEXT: v_readlane_b32 s96, v32, 30
+; SI-NEXT: v_readlane_b32 s87, v32, 29
+; SI-NEXT: v_readlane_b32 s86, v32, 28
+; SI-NEXT: v_readlane_b32 s85, v32, 27
+; SI-NEXT: v_readlane_b32 s84, v32, 26
+; SI-NEXT: v_readlane_b32 s83, v32, 25
+; SI-NEXT: v_readlane_b32 s82, v32, 24
+; SI-NEXT: v_readlane_b32 s81, v32, 23
+; SI-NEXT: v_readlane_b32 s80, v32, 22
+; SI-NEXT: v_readlane_b32 s71, v32, 21
+; SI-NEXT: v_readlane_b32 s70, v32, 20
+; SI-NEXT: v_readlane_b32 s69, v32, 19
+; SI-NEXT: v_readlane_b32 s68, v32, 18
+; SI-NEXT: v_readlane_b32 s67, v32, 17
+; SI-NEXT: v_readlane_b32 s66, v32, 16
+; SI-NEXT: v_readlane_b32 s65, v32, 15
+; SI-NEXT: v_readlane_b32 s64, v32, 14
+; SI-NEXT: v_readlane_b32 s55, v32, 13
+; SI-NEXT: v_readlane_b32 s54, v32, 12
+; SI-NEXT: v_readlane_b32 s53, v32, 11
+; SI-NEXT: v_readlane_b32 s52, v32, 10
+; SI-NEXT: v_readlane_b32 s51, v32, 9
+; SI-NEXT: v_readlane_b32 s50, v32, 8
+; SI-NEXT: v_readlane_b32 s49, v32, 7
+; SI-NEXT: v_readlane_b32 s48, v32, 6
+; SI-NEXT: v_readlane_b32 s39, v32, 5
+; SI-NEXT: v_readlane_b32 s38, v32, 4
+; SI-NEXT: v_readlane_b32 s37, v32, 3
+; SI-NEXT: v_readlane_b32 s36, v32, 2
+; SI-NEXT: v_readlane_b32 s35, v32, 1
+; SI-NEXT: v_readlane_b32 s34, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index c9af784cf88bc..5c5e967c87ce4 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2404,10 +2404,6 @@ define <8 x bfloat> @bitcast_v4i32_to_v8bf16(<4 x i32> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr4
@@ -2415,52 +2411,35 @@ define <8 x bfloat> @bitcast_v4i32_to_v8bf16(<4 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB20_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: .LBB20_4: ; %end
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v6
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i32_to_v8bf16:
@@ -2535,61 +2514,49 @@ define inreg <8 x bfloat> @bitcast_v4i32_to_v8bf16_scalar(<4 x i32> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s20, 0
; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s19, 16
-; SI-NEXT: s_and_b32 s9, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s18, 16
-; SI-NEXT: s_and_b32 s11, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s17, 16
-; SI-NEXT: s_and_b32 s13, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB21_3
; SI-NEXT: .LBB21_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB21_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB21_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_and_b32 s7, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s19, 16
-; SI-NEXT: s_and_b32 s9, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s18, 16
-; SI-NEXT: s_and_b32 s11, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s17, 16
-; SI-NEXT: s_and_b32 s13, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s16, 16
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: .LBB21_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s7, s4
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i32_to_v8bf16_scalar:
@@ -2691,23 +2658,19 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v4i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v5, v2
+; SI-NEXT: v_mov_b32_e32 v6, v1
+; SI-NEXT: v_mov_b32_e32 v7, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -2719,45 +2682,41 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB22_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v11, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v9, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v7, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v10
+; SI-NEXT: v_or_b32_e32 v2, v2, v9
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB22_2
; SI-NEXT: .LBB22_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v11
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v9
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
@@ -3123,69 +3082,67 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v8bf16_to_v4i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s18, 16
+; SI-NEXT: s_lshr_b32 s12, s17, 16
+; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
-; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v15
-; SI-NEXT: v_lshr_b64 v[1:2], v[9:10], 16
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v14
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
-; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s12, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB23_3
; SI-NEXT: .LBB23_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
; SI-NEXT: .LBB23_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB23_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB23_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8bf16_to_v4i32_scalar:
@@ -7154,10 +7111,6 @@ define <8 x bfloat> @bitcast_v4f32_to_v8bf16(<4 x float> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr4
@@ -7165,52 +7118,35 @@ define <8 x bfloat> @bitcast_v4f32_to_v8bf16(<4 x float> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB44_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
+; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: .LBB44_4: ; %end
+; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v6
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4f32_to_v8bf16:
@@ -7283,77 +7219,61 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s20, 0
; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s6, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s19, 16
-; SI-NEXT: s_and_b32 s8, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s18, 16
-; SI-NEXT: s_and_b32 s10, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s17, 16
-; SI-NEXT: s_and_b32 s12, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s11, s19, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB45_3
; SI-NEXT: .LBB45_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
; SI-NEXT: .LBB45_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB45_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
+; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
+; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; SI-NEXT: s_branch .LBB45_6
; SI-NEXT: .LBB45_5:
-; SI-NEXT: v_mov_b32_e32 v0, s13
-; SI-NEXT: v_mov_b32_e32 v1, s12
-; SI-NEXT: v_mov_b32_e32 v2, s11
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: v_mov_b32_e32 v7, s10
-; SI-NEXT: v_mov_b32_e32 v3, s9
-; SI-NEXT: v_mov_b32_e32 v6, s8
-; SI-NEXT: v_mov_b32_e32 v4, s7
+; SI-NEXT: v_mov_b32_e32 v6, s11
+; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s6
; SI-NEXT: .LBB45_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[7:8], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[6:7], 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[4:5], 16
-; SI-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: bitcast_v4f32_to_v8bf16_scalar:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s20, 0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v5
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v4f32_to_v8bf16_scalar:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s20, 0
; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_mov_b64 s[4:5], 0
@@ -7451,23 +7371,19 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v4f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v5, v2
+; SI-NEXT: v_mov_b32_e32 v6, v1
+; SI-NEXT: v_mov_b32_e32 v7, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -7479,45 +7395,41 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB46_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v11, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v9, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v7, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v10
+; SI-NEXT: v_or_b32_e32 v2, v2, v9
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB46_2
; SI-NEXT: .LBB46_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v11
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v9
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
@@ -7883,69 +7795,67 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; SI-LABEL: bitcast_v8bf16_to_v4f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s18, 16
+; SI-NEXT: s_lshr_b32 s12, s17, 16
+; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
-; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v15
-; SI-NEXT: v_lshr_b64 v[1:2], v[9:10], 16
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v14
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
-; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s12, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB47_3
; SI-NEXT: .LBB47_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
; SI-NEXT: .LBB47_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB47_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB47_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8bf16_to_v4f32_scalar:
@@ -11516,10 +11426,6 @@ define <8 x bfloat> @bitcast_v2i64_to_v8bf16(<2 x i64> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr4
@@ -11527,52 +11433,35 @@ define <8 x bfloat> @bitcast_v2i64_to_v8bf16(<2 x i64> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB64_4
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: .LBB64_4: ; %end
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v6
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2i64_to_v8bf16:
@@ -11648,61 +11537,49 @@ define inreg <8 x bfloat> @bitcast_v2i64_to_v8bf16_scalar(<2 x i64> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s20, 0
; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s19, 16
-; SI-NEXT: s_and_b32 s9, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s18, 16
-; SI-NEXT: s_and_b32 s11, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s17, 16
-; SI-NEXT: s_and_b32 s13, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB65_3
; SI-NEXT: .LBB65_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB65_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB65_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_u32 s4, s16, 3
-; SI-NEXT: s_addc_u32 s5, s17, 0
-; SI-NEXT: s_add_u32 s8, s18, 3
-; SI-NEXT: s_addc_u32 s6, s19, 0
-; SI-NEXT: s_and_b32 s7, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s9, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_and_b32 s11, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s5, 16
-; SI-NEXT: s_and_b32 s13, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s4, 16
+; SI-NEXT: s_add_u32 s18, s18, 3
+; SI-NEXT: s_addc_u32 s19, s19, 0
+; SI-NEXT: s_add_u32 s16, s16, 3
+; SI-NEXT: s_addc_u32 s17, s17, 0
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: .LBB65_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s7, s4
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2i64_to_v8bf16_scalar:
@@ -11804,23 +11681,19 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v2i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v5, v2
+; SI-NEXT: v_mov_b32_e32 v6, v1
+; SI-NEXT: v_mov_b32_e32 v7, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -11832,45 +11705,41 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB66_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v11, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v9, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v7, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v10
+; SI-NEXT: v_or_b32_e32 v2, v2, v9
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB66_2
; SI-NEXT: .LBB66_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v11
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v9
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
@@ -12236,69 +12105,67 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v8bf16_to_v2i64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s18, 16
+; SI-NEXT: s_lshr_b32 s12, s17, 16
+; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
-; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v15
-; SI-NEXT: v_lshr_b64 v[1:2], v[9:10], 16
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v14
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
-; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s12, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB67_3
; SI-NEXT: .LBB67_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
; SI-NEXT: .LBB67_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB67_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB67_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB67_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8bf16_to_v2i64_scalar:
@@ -15432,10 +15299,6 @@ define <8 x bfloat> @bitcast_v2f64_to_v8bf16(<2 x double> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr4
@@ -15443,48 +15306,33 @@ define <8 x bfloat> @bitcast_v2f64_to_v8bf16(<2 x double> %a, i32 %b) #0 {
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB80_4
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; SI-NEXT: .LBB80_4: ; %end
+; SI-NEXT: v_alignbit_b32 v4, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v6
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2f64_to_v8bf16:
@@ -15556,69 +15404,53 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s20, 0
; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s13, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s19, 16
-; SI-NEXT: s_and_b32 s11, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s18, 16
-; SI-NEXT: s_and_b32 s9, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s17, 16
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s11, s19, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB81_3
; SI-NEXT: .LBB81_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
; SI-NEXT: .LBB81_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB81_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f64 v[0:1], s[18:19], 1.0
-; SI-NEXT: v_add_f64 v[8:9], s[16:17], 1.0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; SI-NEXT: s_branch .LBB81_6
; SI-NEXT: .LBB81_5:
-; SI-NEXT: v_mov_b32_e32 v5, s13
-; SI-NEXT: v_mov_b32_e32 v4, s12
-; SI-NEXT: v_mov_b32_e32 v6, s11
-; SI-NEXT: v_mov_b32_e32 v3, s10
-; SI-NEXT: v_mov_b32_e32 v7, s9
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v7, s11
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v5, s6
+; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: .LBB81_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[7:8], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[6:7], 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[4:5], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; SI-NEXT: v_or_b32_e32 v1, v1, v5
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2f64_to_v8bf16_scalar:
@@ -15716,23 +15548,19 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v2f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v5, v2
+; SI-NEXT: v_mov_b32_e32 v6, v1
+; SI-NEXT: v_mov_b32_e32 v7, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -15744,45 +15572,41 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB82_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v11, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v9, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v7, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v10
+; SI-NEXT: v_or_b32_e32 v2, v2, v9
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB82_2
; SI-NEXT: .LBB82_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v11
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v9
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
@@ -16148,69 +15972,67 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; SI-LABEL: bitcast_v8bf16_to_v2f64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s19, 16
+; SI-NEXT: s_lshr_b32 s11, s18, 16
+; SI-NEXT: s_lshr_b32 s12, s17, 16
+; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
-; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v15
-; SI-NEXT: v_lshr_b64 v[1:2], v[9:10], 16
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v14
-; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
-; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s12, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB83_3
; SI-NEXT: .LBB83_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
; SI-NEXT: .LBB83_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB83_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB83_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB83_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8bf16_to_v2f64_scalar:
@@ -18882,77 +18704,82 @@ define <8 x bfloat> @bitcast_v8i16_to_v8bf16(<8 x i16> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8i16_to_v8bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v7
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB92_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v3
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v8, v1, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v4, v1, v15
+; SI-NEXT: v_or_b32_e32 v9, v0, v12
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_alignbit_b32 v10, v8, v12, 16
+; SI-NEXT: v_alignbit_b32 v11, v4, v14, 16
+; SI-NEXT: v_or_b32_e32 v7, v0, v14
; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; %bb.2: ; %Flow
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: .LBB92_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB92_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v3, v7, v3
+; SI-NEXT: v_or_b32_e32 v2, v14, v2
+; SI-NEXT: v_or_b32_e32 v0, v12, v0
+; SI-NEXT: v_add_i32_e32 v7, vcc, 0x30000, v2
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v3
+; SI-NEXT: v_add_i32_e32 v9, vcc, 0x30000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_mov_b32 s6, 0x30000
-; SI-NEXT: v_or_b32_e32 v2, v6, v2
-; SI-NEXT: v_or_b32_e32 v1, v5, v1
-; SI-NEXT: v_or_b32_e32 v0, v4, v0
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x30000, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x30000, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x30000, v0
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_or_b32_e32 v2, v15, v2
+; SI-NEXT: v_or_b32_e32 v0, v13, v0
+; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v2
+; SI-NEXT: v_add_i32_e32 v8, vcc, 0x30000, v0
+; SI-NEXT: v_alignbit_b32 v10, v8, v9, 16
+; SI-NEXT: v_alignbit_b32 v11, v4, v7, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; SI-NEXT: .LBB92_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v9
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v11
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i16_to_v8bf16:
@@ -19034,84 +18861,84 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; SI-LABEL: bitcast_v8i16_to_v8bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s22, s19, 16
-; SI-NEXT: s_lshr_b32 s21, s18, 16
-; SI-NEXT: s_lshr_b32 s15, s17, 16
-; SI-NEXT: s_lshr_b32 s14, s16, 16
+; SI-NEXT: s_lshr_b32 s15, s19, 16
+; SI-NEXT: s_lshr_b32 s22, s18, 16
+; SI-NEXT: s_lshr_b32 s14, s17, 16
+; SI-NEXT: s_lshr_b32 s21, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s10, s16, 16
-; SI-NEXT: s_lshl_b32 s13, s14, 16
-; SI-NEXT: s_lshl_b32 s8, s17, 16
-; SI-NEXT: s_lshl_b32 s12, s15, 16
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_lshl_b32 s11, s21, 16
-; SI-NEXT: s_lshl_b32 s6, s19, 16
-; SI-NEXT: s_lshl_b32 s9, s22, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s14, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s24, s21, 16
+; SI-NEXT: s_or_b32 s25, s5, s7
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s15, 16
+; SI-NEXT: s_or_b32 s6, s4, s24
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s22, 16
+; SI-NEXT: s_or_b32 s27, s5, s7
+; SI-NEXT: s_or_b32 s4, s4, s26
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_mov_b32 s7, s25
+; SI-NEXT: s_mov_b32 s5, s27
; SI-NEXT: s_branch .LBB93_3
; SI-NEXT: .LBB93_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB93_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
; SI-NEXT: s_cbranch_scc1 .LBB93_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s22, 16
; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s22, 16
+; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s21, 16
-; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s15, 16
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s15, 16
; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s21, 16
+; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s16, 0xffff
+; SI-NEXT: s_and_b32 s7, s17, 0xffff
; SI-NEXT: s_lshl_b32 s8, s14, 16
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: s_and_b32 s13, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s7, 16
-; SI-NEXT: s_and_b32 s12, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s6, 16
-; SI-NEXT: s_and_b32 s11, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s5, 16
-; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s14, s7, 16
+; SI-NEXT: s_lshr_b32 s15, s5, 16
; SI-NEXT: .LBB93_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s12
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s6, s6, s8
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s14, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s4, s4, s8
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s15, 16
+; SI-NEXT: s_or_b32 s5, s5, s8
+; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i16_to_v8bf16_scalar:
@@ -19230,95 +19057,57 @@ define <8 x i16> @bitcast_v8bf16_to_v8i16(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v8i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v5
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v10
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB94_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v1
+; SI-NEXT: s_cbranch_execz .LBB94_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v10
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v11
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v8
-; SI-NEXT: v_alignbit_b32 v3, v4, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_alignbit_b32 v6, v1, v6, 16
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_alignbit_b32 v0, v8, v0, 16
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_alignbit_b32 v3, v5, v3, 16
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; SI-NEXT: v_alignbit_b32 v1, v6, v1, 16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_alignbit_b32 v2, v8, v2, 16
+; SI-NEXT: v_alignbit_b32 v8, v1, v4, 16
; SI-NEXT: v_alignbit_b32 v7, v3, v7, 16
-; SI-NEXT: .LBB94_4: ; %end
+; SI-NEXT: .LBB94_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v1, v1, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v4
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: v_or_b32_e32 v2, v2, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -19668,91 +19457,74 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v8bf16_to_v8i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s7, s19, 16
+; SI-NEXT: s_lshr_b32 s8, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
; SI-NEXT: s_branch .LBB95_3
; SI-NEXT: .LBB95_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: .LBB95_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB95_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v5, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v5
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; SI-NEXT: v_lshr_b64 v[3:4], v[8:9], 16
-; SI-NEXT: v_lshr_b64 v[5:6], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
-; SI-NEXT: v_lshr_b64 v[12:13], v[4:5], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[2:3], 16
-; SI-NEXT: .LBB95_5: ; %end
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_add_f32_e32 v6, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s4, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; SI-NEXT: v_lshr_b64 v[3:4], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[5:6], v[9:10], 16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
+; SI-NEXT: v_lshr_b64 v[13:14], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[2:3], 16
+; SI-NEXT: s_branch .LBB95_6
+; SI-NEXT: .LBB95_5:
+; SI-NEXT: v_mov_b32_e32 v10, s6
+; SI-NEXT: v_mov_b32_e32 v7, s7
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v8, s18
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v13, s9
+; SI-NEXT: v_mov_b32_e32 v11, s8
+; SI-NEXT: .LBB95_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v13
; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v10
; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v13
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -21866,92 +21638,93 @@ define <8 x bfloat> @bitcast_v8f16_to_v8bf16(<8 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8f16_to_v8bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB100_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v15
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; SI-NEXT: v_or_b32_e32 v8, v1, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v13
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v9, v3, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_or_b32_e32 v4, v0, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_alignbit_b32 v10, v8, v1, 16
+; SI-NEXT: v_alignbit_b32 v11, v9, v3, 16
+; SI-NEXT: v_or_b32_e32 v7, v0, v3
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; %bb.2: ; %Flow
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: .LBB100_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v11, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v4
; SI-NEXT: v_cvt_f32_f16_e32 v4, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v12
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_or_b32_e32 v4, v0, v10
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_or_b32_e32 v7, v2, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v6
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v9, v2, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_or_b32_e32 v8, v0, v1
+; SI-NEXT: v_alignbit_b32 v10, v8, v10, 16
+; SI-NEXT: v_alignbit_b32 v11, v9, v11, 16
; SI-NEXT: .LBB100_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8f16_to_v8bf16:
@@ -22034,98 +21807,100 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; SI-LABEL: bitcast_v8f16_to_v8bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s9, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s7, s17, 16
-; SI-NEXT: s_lshr_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s15, s19, 16
+; SI-NEXT: s_lshr_b32 s21, s18, 16
+; SI-NEXT: s_lshr_b32 s14, s17, 16
+; SI-NEXT: s_lshr_b32 s22, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s10, s16, 16
-; SI-NEXT: s_lshl_b32 s11, s6, 16
-; SI-NEXT: s_lshl_b32 s12, s17, 16
-; SI-NEXT: s_lshl_b32 s13, s7, 16
-; SI-NEXT: s_lshl_b32 s14, s18, 16
-; SI-NEXT: s_lshl_b32 s15, s8, 16
-; SI-NEXT: s_lshl_b32 s20, s19, 16
-; SI-NEXT: s_lshl_b32 s21, s9, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: s_or_b32 s20, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s21, 16
+; SI-NEXT: s_or_b32 s23, s5, s8
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s14, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s15, 16
+; SI-NEXT: s_or_b32 s9, s4, s7
+; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB101_3
; SI-NEXT: .LBB101_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr23
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: .LBB101_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB101_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s8
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s16
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s18
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s7
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
-; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s22
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s21
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v9, v1, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s15
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s19
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s17
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v8, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v9, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s14
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v10, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
+; SI-NEXT: v_or_b32_e32 v2, v11, v2
; SI-NEXT: s_branch .LBB101_6
; SI-NEXT: .LBB101_5:
-; SI-NEXT: v_mov_b32_e32 v5, s21
-; SI-NEXT: v_mov_b32_e32 v4, s20
-; SI-NEXT: v_mov_b32_e32 v6, s15
-; SI-NEXT: v_mov_b32_e32 v3, s14
-; SI-NEXT: v_mov_b32_e32 v7, s13
-; SI-NEXT: v_mov_b32_e32 v2, s12
-; SI-NEXT: v_mov_b32_e32 v1, s11
-; SI-NEXT: v_mov_b32_e32 v0, s10
+; SI-NEXT: v_mov_b32_e32 v8, s15
+; SI-NEXT: v_mov_b32_e32 v10, s14
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: v_mov_b32_e32 v2, s23
+; SI-NEXT: v_mov_b32_e32 v9, s20
+; SI-NEXT: v_mov_b32_e32 v6, s6
+; SI-NEXT: v_mov_b32_e32 v4, s10
; SI-NEXT: .LBB101_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[7:8], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[6:7], 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[4:5], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT: v_or_b32_e32 v1, v1, v5
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8f16_to_v8bf16_scalar:
@@ -22247,101 +22022,85 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v8f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v2
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v7
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_alignbit_b32 v5, v1, v9, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
-; SI-NEXT: v_alignbit_b32 v6, v5, v0, 16
-; SI-NEXT: v_alignbit_b32 v4, v3, v12, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
-; SI-NEXT: v_alignbit_b32 v7, v4, v0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
-; SI-NEXT: v_alignbit_b32 v0, v0, v15, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v13, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v9, v1, v12
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v7, v1, v14
+; SI-NEXT: v_or_b32_e32 v4, v0, v15
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_alignbit_b32 v10, v9, v15, 16
+; SI-NEXT: v_alignbit_b32 v11, v7, v13, 16
+; SI-NEXT: v_or_b32_e32 v8, v0, v13
+; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: .LBB102_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v15
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v10
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v11
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v4, v3, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v10
+; SI-NEXT: v_alignbit_b32 v4, v4, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v13
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_alignbit_b32 v8, v5, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v14
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; SI-NEXT: v_alignbit_b32 v7, v5, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v12
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v5, v1, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_alignbit_b32 v6, v5, v6, 16
-; SI-NEXT: v_alignbit_b32 v7, v4, v7, 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; SI-NEXT: v_alignbit_b32 v9, v6, v1, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
+; SI-NEXT: v_alignbit_b32 v10, v9, v1, 16
+; SI-NEXT: v_alignbit_b32 v11, v7, v0, 16
; SI-NEXT: .LBB102_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_or_b32_e32 v1, v5, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: v_or_b32_e32 v2, v2, v5
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8bf16_to_v8f16:
@@ -22693,93 +22452,94 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; SI-LABEL: bitcast_v8bf16_to_v8f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s15, s19, 16
+; SI-NEXT: s_lshr_b32 s21, s18, 16
+; SI-NEXT: s_lshr_b32 s14, s17, 16
+; SI-NEXT: s_lshr_b32 s22, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v21
-; SI-NEXT: v_lshr_b64 v[14:15], v[4:5], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v22
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshr_b64 v[18:19], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[9:10], v[16:17], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[11:12], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: s_or_b32 s20, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s21, 16
+; SI-NEXT: s_or_b32 s23, s5, s8
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s14, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s15, 16
+; SI-NEXT: s_or_b32 s9, s4, s7
+; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB103_3
; SI-NEXT: .LBB103_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: .LBB103_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB103_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v23
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v14
-; SI-NEXT: v_lshr_b64 v[9:10], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_lshr_b64 v[12:13], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[3:4], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
-; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v14
-; SI-NEXT: v_lshr_b64 v[14:15], v[4:5], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[2:3], 16
-; SI-NEXT: .LBB103_5: ; %end
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v18
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v5, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v5
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_add_f32_e32 v6, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v5
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s4, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; SI-NEXT: v_lshr_b64 v[3:4], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[5:6], v[9:10], 16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
+; SI-NEXT: v_lshr_b64 v[13:14], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[2:3], 16
+; SI-NEXT: s_branch .LBB103_6
+; SI-NEXT: .LBB103_5:
+; SI-NEXT: v_mov_b32_e32 v10, s14
+; SI-NEXT: v_mov_b32_e32 v7, s15
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: v_mov_b32_e32 v8, s23
+; SI-NEXT: v_mov_b32_e32 v0, s20
+; SI-NEXT: v_mov_b32_e32 v13, s6
+; SI-NEXT: v_mov_b32_e32 v11, s10
+; SI-NEXT: .LBB103_6: ; %end
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v13
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -24939,30 +24699,25 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v8bf16_to_v16i8:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v16, v3
+; SI-NEXT: v_mov_b32_e32 v18, v2
+; SI-NEXT: v_mov_b32_e32 v17, v1
+; SI-NEXT: v_mov_b32_e32 v19, v0
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v19
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v14
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: ; implicit-def: $vgpr9
@@ -24970,7 +24725,6 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -24982,56 +24736,52 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB108_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v17
-; SI-NEXT: v_alignbit_b32 v0, v0, v20, 16
-; SI-NEXT: v_alignbit_b32 v4, v6, v18, 16
-; SI-NEXT: v_alignbit_b32 v8, v5, v23, 16
-; SI-NEXT: v_alignbit_b32 v12, v14, v21, 16
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v18
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v8, v5, v23
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v22
+; SI-NEXT: v_or_b32_e32 v4, v1, v20
+; SI-NEXT: v_or_b32_e32 v12, v5, v21
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
; SI-NEXT: v_alignbit_b32 v11, v12, v8, 24
; SI-NEXT: v_alignbit_b32 v10, v12, v8, 16
; SI-NEXT: v_alignbit_b32 v9, v12, v8, 8
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v16
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v12
+; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
+; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB108_2
; SI-NEXT: .LBB108_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v23
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_alignbit_b32 v8, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v16
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v21
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v15
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
; SI-NEXT: v_alignbit_b32 v12, v14, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v19
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v22
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v20
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
; SI-NEXT: v_alignbit_b32 v4, v6, v1, 16
@@ -25594,109 +25344,117 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v8bf16_to_v16i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s19, 16
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
+; SI-NEXT: s_lshr_b32 s28, s19, 16
+; SI-NEXT: s_lshr_b32 s29, s18, 16
+; SI-NEXT: s_lshr_b32 s26, s17, 16
+; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v30
-; SI-NEXT: v_lshr_b64 v[22:23], v[27:28], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v31
-; SI-NEXT: v_lshr_b64 v[0:1], v[25:26], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v29
-; SI-NEXT: v_lshr_b64 v[23:24], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
-; SI-NEXT: v_lshr_b64 v[20:21], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 8
-; SI-NEXT: v_lshr_b64 v[17:18], v[22:23], 24
-; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 8
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v29
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v31
-; SI-NEXT: v_lshrrev_b32_e32 v19, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v23
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s27, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s26, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: s_and_b32 s7, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s29, 16
+; SI-NEXT: s_or_b32 s12, s7, s9
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s28, 16
+; SI-NEXT: s_or_b32 s13, s7, s9
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 8
+; SI-NEXT: s_lshr_b64 s[14:15], s[12:13], 24
+; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[22:23], s[12:13], 8
+; SI-NEXT: s_lshr_b32 s7, s5, 8
+; SI-NEXT: s_lshr_b32 s9, s13, 8
+; SI-NEXT: s_bfe_u32 s11, s26, 0x80008
+; SI-NEXT: s_bfe_u32 s15, s28, 0x80008
+; SI-NEXT: s_mov_b64 s[24:25], 0
; SI-NEXT: s_branch .LBB109_3
; SI-NEXT: .LBB109_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: s_mov_b64 s[24:25], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr15
; SI-NEXT: .LBB109_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[24:25], s[24:25], exec
+; SI-NEXT: s_cselect_b32 s21, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s21, 1
; SI-NEXT: s_cbranch_scc1 .LBB109_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s28, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[22:23], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v31
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v10, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s26, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v7, s4, v2
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_lshr_b64 v[23:24], v[13:14], 16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v10
-; SI-NEXT: v_lshr_b64 v[10:11], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
-; SI-NEXT: v_lshr_b64 v[20:21], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[17:18], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[18:19], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 8
-; SI-NEXT: v_lshr_b64 v[17:18], v[22:23], 24
-; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 8
-; SI-NEXT: v_lshrrev_b32_e32 v19, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v23
+; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v10
+; SI-NEXT: v_lshr_b64 v[11:12], v[16:17], 24
+; SI-NEXT: v_lshr_b64 v[19:20], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[9:10], v[16:17], 8
+; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v17
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; SI-NEXT: .LBB109_5: ; %end
+; SI-NEXT: s_branch .LBB109_6
+; SI-NEXT: .LBB109_5:
+; SI-NEXT: v_mov_b32_e32 v15, s15
+; SI-NEXT: v_mov_b32_e32 v7, s11
+; SI-NEXT: v_mov_b32_e32 v6, s26
+; SI-NEXT: v_mov_b32_e32 v14, s28
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v17, s13
+; SI-NEXT: v_mov_b32_e32 v16, s12
+; SI-NEXT: v_mov_b32_e32 v5, s7
+; SI-NEXT: v_mov_b32_e32 v13, s9
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v18, s8
+; SI-NEXT: v_mov_b32_e32 v8, s10
+; SI-NEXT: v_mov_b32_e32 v11, s14
+; SI-NEXT: v_mov_b32_e32 v19, s20
+; SI-NEXT: v_mov_b32_e32 v9, s22
+; SI-NEXT: .LBB109_6: ; %end
; SI-NEXT: v_mov_b32_e32 v4, v1
; SI-NEXT: v_mov_b32_e32 v1, v8
-; SI-NEXT: v_mov_b32_e32 v2, v20
-; SI-NEXT: v_mov_b32_e32 v9, v11
-; SI-NEXT: v_mov_b32_e32 v5, v19
-; SI-NEXT: v_mov_b32_e32 v8, v22
-; SI-NEXT: v_mov_b32_e32 v11, v17
-; SI-NEXT: v_mov_b32_e32 v12, v23
-; SI-NEXT: v_mov_b32_e32 v13, v16
+; SI-NEXT: v_mov_b32_e32 v2, v18
+; SI-NEXT: v_mov_b32_e32 v10, v19
+; SI-NEXT: v_mov_b32_e32 v8, v16
+; SI-NEXT: v_mov_b32_e32 v12, v17
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8bf16_to_v16i8_scalar:
@@ -26325,144 +26083,142 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v3
-; SI-NEXT: v_lshlrev_b32_e32 v21, 8, v5
-; SI-NEXT: v_lshlrev_b32_e32 v18, 24, v7
-; SI-NEXT: v_lshlrev_b32_e32 v20, 24, v11
-; SI-NEXT: v_lshlrev_b32_e32 v23, 8, v13
-; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v15
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v5
+; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v7
+; SI-NEXT: v_lshlrev_b32_e32 v18, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v21, 8, v13
+; SI-NEXT: v_lshlrev_b32_e32 v20, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v11
+; SI-NEXT: v_lshlrev_b32_e32 v13, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v23, 8, v9
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT: v_or_b32_e32 v3, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v5, v16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v21
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v11, v18, v0
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v14
+; SI-NEXT: v_or_b32_e32 v0, v0, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v4, v16, v3
+; SI-NEXT: v_or_b32_e32 v2, v18, v2
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v17
+; SI-NEXT: v_or_b32_e32 v3, v3, v21
+; SI-NEXT: v_or_b32_e32 v6, v20, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v10
+; SI-NEXT: v_or_b32_e32 v9, v0, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v9
-; SI-NEXT: v_or_b32_e32 v13, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v15, v20, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v12
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v0, v0, v23
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v19, v22, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_or_b32_e32 v3, v3, v6
+; SI-NEXT: v_or_b32_e32 v5, v22, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v7, v1, v2, 16
+; SI-NEXT: v_alignbit_b32 v11, v3, v5, 16
+; SI-NEXT: v_or_b32_e32 v5, v0, v5
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v6
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: .LBB110_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v12
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v8
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v10
+; SI-NEXT: v_or_b32_e32 v1, v23, v1
; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v14
-; SI-NEXT: v_or_b32_e32 v3, v23, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x300, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v22, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, 0x300, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v22, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: s_mov_b32 s7, 0x3000000
-; SI-NEXT: v_add_i32_e32 v12, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v8
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v9
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v10
+; SI-NEXT: v_add_i32_e32 v5, vcc, s7, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v12
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v14
; SI-NEXT: s_movk_i32 s6, 0x300
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v20, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
+; SI-NEXT: v_or_b32_e32 v1, v21, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v20, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_add_i32_e32 v8, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v4
+; SI-NEXT: v_add_i32_e32 v3, vcc, s7, v1
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v6
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v2
-; SI-NEXT: v_or_b32_e32 v3, v21, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v0, v13, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v18, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v9, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v6
+; SI-NEXT: v_or_b32_e32 v0, v17, v0
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v4, v18, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v1, v16, v1
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_add_i32_e32 v4, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x3000000, v0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v8
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v12
+; SI-NEXT: v_add_i32_e32 v1, vcc, 0x3000000, v0
+; SI-NEXT: v_alignbit_b32 v7, v1, v9, 16
+; SI-NEXT: v_alignbit_b32 v11, v3, v5, 16
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v3
; SI-NEXT: .LBB110_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v13
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v17
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v15
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v19
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i8_to_v8bf16:
@@ -26867,138 +26623,142 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v2
-; SI-NEXT: v_readfirstlane_b32 s12, v1
+; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s14, v0
+; SI-NEXT: v_readfirstlane_b32 s15, v0
; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_or_b32 s6, s5, s4
-; SI-NEXT: s_and_b32 s4, s18, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s19, 24
-; SI-NEXT: s_or_b32 s8, s5, s4
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s21, 8
+; SI-NEXT: s_lshl_b32 s5, s17, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s7, s4, 16
-; SI-NEXT: s_and_b32 s4, s22, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s23, 24
-; SI-NEXT: s_or_b32 s10, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s19, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s40, s6, s5
+; SI-NEXT: s_or_b32 s6, s4, s40
; SI-NEXT: s_and_b32 s4, s24, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s25, 24
-; SI-NEXT: s_or_b32 s9, s5, s4
-; SI-NEXT: s_and_b32 s4, s26, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s27, 24
-; SI-NEXT: s_or_b32 s13, s5, s4
-; SI-NEXT: s_and_b32 s4, s28, 0xff
-; SI-NEXT: s_lshl_b32 s5, s29, 8
+; SI-NEXT: s_lshl_b32 s5, s25, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s11, s4, 16
-; SI-NEXT: s_and_b32 s4, s14, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s12, 24
-; SI-NEXT: s_or_b32 s15, s5, s4
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s26, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s7, s27, 24
+; SI-NEXT: s_or_b32 s42, s7, s5
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s7, s21, 8
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_and_b32 s7, s22, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s8, s23, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_or_b32 s41, s5, s7
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
+; SI-NEXT: s_and_b32 s5, s28, 0xff
+; SI-NEXT: s_lshl_b32 s9, s29, 8
+; SI-NEXT: s_or_b32 s5, s5, s9
+; SI-NEXT: s_and_b32 s9, s15, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_lshl_b32 s10, s14, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_or_b32 s43, s5, s9
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshr_b64 s[10:11], s[42:43], 16
+; SI-NEXT: s_or_b32 s4, s4, s42
+; SI-NEXT: s_lshr_b32 s11, s7, 16
+; SI-NEXT: s_lshr_b32 s9, s9, 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_mov_b32 s7, s41
+; SI-NEXT: s_mov_b32 s5, s43
; SI-NEXT: s_branch .LBB111_3
; SI-NEXT: .LBB111_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr15
; SI-NEXT: .LBB111_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
; SI-NEXT: s_cbranch_scc1 .LBB111_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_and_b32 s4, s28, 0xff
-; SI-NEXT: s_lshl_b32 s5, s29, 8
-; SI-NEXT: s_add_i32 s14, s14, 3
+; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_and_b32 s4, s24, 0xff
+; SI-NEXT: s_lshl_b32 s5, s25, 8
+; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s14, 0xff
+; SI-NEXT: s_and_b32 s6, s26, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s12, 24
+; SI-NEXT: s_lshl_b32 s5, s27, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s24, 0xff
-; SI-NEXT: s_lshl_b32 s6, s25, 8
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_and_b32 s5, s28, 0xff
+; SI-NEXT: s_lshl_b32 s6, s29, 8
+; SI-NEXT: s_add_i32 s15, s15, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s7, s26, 0xff
+; SI-NEXT: s_and_b32 s7, s15, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s6, s27, 24
+; SI-NEXT: s_lshl_b32 s6, s14, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s20, 0xff
-; SI-NEXT: s_lshl_b32 s7, s21, 8
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_and_b32 s6, s16, 0xff
+; SI-NEXT: s_lshl_b32 s7, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s8, s22, 0xff
+; SI-NEXT: s_and_b32 s8, s18, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s23, 24
+; SI-NEXT: s_lshl_b32 s7, s19, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s7, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s16, 0xff
-; SI-NEXT: s_lshl_b32 s8, s17, 8
-; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_and_b32 s9, s18, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s8, s19, 24
+; SI-NEXT: s_and_b32 s7, s20, 0xff
+; SI-NEXT: s_lshl_b32 s8, s21, 8
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s9, s22, 0xff
+; SI-NEXT: s_addk_i32 s7, 0x300
+; SI-NEXT: s_lshl_b32 s8, s23, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: s_and_b32 s8, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s10, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s13, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s5, 16
-; SI-NEXT: s_and_b32 s15, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s4, 16
+; SI-NEXT: s_add_i32 s7, s7, 0x3000000
+; SI-NEXT: s_lshr_b64 s[8:9], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s11, s7, 16
+; SI-NEXT: s_lshr_b32 s9, s5, 16
; SI-NEXT: .LBB111_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s11
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s6, s6, s8
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s11, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s10, 16
+; SI-NEXT: s_or_b32 s4, s4, s8
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s9, 16
+; SI-NEXT: s_or_b32 s5, s5, s8
+; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i8_to_v8bf16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index ebc3b81e62385..6a86f63bb1c68 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -444,16 +444,13 @@ define bfloat @bitcast_i16_to_bf16(i16 %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x30000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: ; %bb.2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_i16_to_bf16:
@@ -542,23 +539,19 @@ define inreg bfloat @bitcast_i16_to_bf16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s7, s6, 16
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB5_3
; SI-NEXT: .LBB5_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB5_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB5_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_lshl_b32 s4, s6, 16
-; SI-NEXT: s_add_i32 s7, s4, 0x30000
+; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: .LBB5_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_i16_to_bf16_scalar:
@@ -644,9 +637,8 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) #0 {
; SI-LABEL: bitcast_bf16_to_i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v0
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -658,12 +650,12 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB6_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB6_2
; SI-NEXT: .LBB6_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -787,27 +779,28 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; SI-LABEL: bitcast_bf16_to_i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshl_b32 s4, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; SI-NEXT: s_and_b32 s6, s16, 0xffff
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB7_3
; SI-NEXT: .LBB7_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: .LBB7_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB7_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v0, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: .LBB7_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB7_5:
+; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_bf16_to_i16_scalar:
@@ -925,24 +918,15 @@ define bfloat @bitcast_f16_to_bf16(half %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB8_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: ; %bb.1: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: .LBB8_4: ; %end
+; SI-NEXT: ; %bb.2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_f16_to_bf16:
@@ -1030,12 +1014,10 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s6, s16, 16
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB9_3
; SI-NEXT: .LBB9_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: .LBB9_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -1045,13 +1027,9 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB9_6
+; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB9_5:
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: .LBB9_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_f16_to_bf16_scalar:
@@ -1162,28 +1140,15 @@ define half @bitcast_bf16_to_f16(bfloat %a, i32 %b) #0 {
; SI-LABEL: bitcast_bf16_to_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v0
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execnz .LBB10_3
-; SI-NEXT: ; %bb.1: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execnz .LBB10_4
-; SI-NEXT: .LBB10_2: ; %end
-; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB10_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB10_2
-; SI-NEXT: .LBB10_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: ; %bb.2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -1305,27 +1270,26 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; SI-LABEL: bitcast_bf16_to_f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshl_b32 s4, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB11_3
; SI-NEXT: .LBB11_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: .LBB11_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB11_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v0, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB11_5:
+; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_bf16_to_f16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 65ce4e7da8a23..43105b38616f6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -3190,112 +3190,71 @@ define <16 x bfloat> @bitcast_v8i32_to_v16bf16(<8 x i32> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB20_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: .LBB20_2: ; %Flow
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB20_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; SI-NEXT: .LBB20_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v14
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v12
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v15
+; SI-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; SI-NEXT: v_or_b32_e32 v4, v4, v9
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v13
+; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i32_to_v16bf16:
@@ -3383,105 +3342,81 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s23, 16
-; SI-NEXT: s_and_b32 s9, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s22, 16
-; SI-NEXT: s_and_b32 s11, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s21, 16
-; SI-NEXT: s_and_b32 s13, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s20, 16
-; SI-NEXT: s_and_b32 s15, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s19, 16
-; SI-NEXT: s_and_b32 s25, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s24, s18, 16
-; SI-NEXT: s_and_b32 s27, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s26, s17, 16
-; SI-NEXT: s_and_b32 s29, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s28, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s21, 16
+; SI-NEXT: s_lshr_b32 s24, s19, 16
+; SI-NEXT: s_lshr_b32 s25, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB21_3
; SI-NEXT: .LBB21_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr25
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr24
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: .LBB21_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB21_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_and_b32 s7, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s23, 16
-; SI-NEXT: s_and_b32 s9, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s22, 16
-; SI-NEXT: s_and_b32 s11, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s21, 16
-; SI-NEXT: s_and_b32 s13, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s20, 16
-; SI-NEXT: s_and_b32 s15, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s19, 16
-; SI-NEXT: s_and_b32 s25, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s24, s18, 16
-; SI-NEXT: s_and_b32 s27, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s26, s17, 16
-; SI-NEXT: s_and_b32 s29, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s28, s16, 16
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s21, 16
+; SI-NEXT: s_lshr_b32 s24, s19, 16
+; SI-NEXT: s_lshr_b32 s25, s17, 16
; SI-NEXT: .LBB21_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s29
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s27
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s26
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s25
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s24
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_and_b32 s7, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s25, 16
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_and_b32 s9, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: s_and_b32 s9, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s24, 16
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s10, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s6, s10, s6
+; SI-NEXT: s_and_b32 s10, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s15, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s11, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s11, s4
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: v_mov_b32_e32 v4, s6
+; SI-NEXT: v_mov_b32_e32 v5, s10
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i32_to_v16bf16_scalar:
@@ -3605,39 +3540,31 @@ define <8 x i32> @bitcast_v16bf16_to_v8i32(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v8i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v16, v7
+; SI-NEXT: v_mov_b32_e32 v9, v6
+; SI-NEXT: v_mov_b32_e32 v10, v5
+; SI-NEXT: v_mov_b32_e32 v11, v4
+; SI-NEXT: v_mov_b32_e32 v12, v3
+; SI-NEXT: v_mov_b32_e32 v13, v2
+; SI-NEXT: v_mov_b32_e32 v14, v1
+; SI-NEXT: v_mov_b32_e32 v15, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v15
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v7
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -3649,85 +3576,77 @@ define <8 x i32> @bitcast_v16bf16_to_v8i32(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB22_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v23, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v21, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v19, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v17, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v15, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v13, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v11, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v9, 16
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v22
+; SI-NEXT: v_or_b32_e32 v2, v2, v21
+; SI-NEXT: v_or_b32_e32 v3, v3, v20
+; SI-NEXT: v_or_b32_e32 v4, v4, v19
+; SI-NEXT: v_or_b32_e32 v5, v5, v18
+; SI-NEXT: v_or_b32_e32 v6, v6, v17
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB22_2
; SI-NEXT: .LBB22_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v22
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v21
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v20
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v14
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v19
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v18
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v10
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v17
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -4351,117 +4270,111 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; SI-LABEL: bitcast_v16bf16_to_v8i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s22, 16
+; SI-NEXT: s_lshr_b32 s25, s21, 16
+; SI-NEXT: s_lshr_b32 s26, s20, 16
+; SI-NEXT: s_lshr_b32 s27, s19, 16
+; SI-NEXT: s_lshr_b32 s28, s18, 16
+; SI-NEXT: s_lshr_b32 s29, s17, 16
+; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
-; SI-NEXT: v_lshr_b64 v[0:1], v[23:24], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[21:22], 16
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v30
-; SI-NEXT: v_lshr_b64 v[2:3], v[19:20], 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v29
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[15:16], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; SI-NEXT: v_lshr_b64 v[5:6], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v26
-; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
-; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s40, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s29, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s28, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s27, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_and_b32 s8, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s26, 16
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s25, 16
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s10, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s15, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB23_3
; SI-NEXT: .LBB23_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
; SI-NEXT: .LBB23_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
; SI-NEXT: s_cbranch_scc1 .LBB23_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v8, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB23_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16bf16_to_v8i32_scalar:
@@ -10650,112 +10563,71 @@ define <16 x bfloat> @bitcast_v8f32_to_v16bf16(<8 x float> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB44_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: .LBB44_2: ; %Flow
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB44_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
+; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_add_f32_e32 v4, 1.0, v4
+; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
; SI-NEXT: v_add_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_add_f32_e32 v6, 1.0, v6
+; SI-NEXT: v_add_f32_e32 v4, 1.0, v4
; SI-NEXT: v_add_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; SI-NEXT: v_add_f32_e32 v6, 1.0, v6
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; SI-NEXT: .LBB44_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v14
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v12
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v15
+; SI-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; SI-NEXT: v_or_b32_e32 v4, v4, v9
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v13
+; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8f32_to_v16bf16:
@@ -10838,123 +10710,91 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s6, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s23, 16
-; SI-NEXT: s_and_b32 s8, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s22, 16
-; SI-NEXT: s_and_b32 s10, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s21, 16
-; SI-NEXT: s_and_b32 s12, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s20, 16
-; SI-NEXT: s_and_b32 s14, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s19, 16
-; SI-NEXT: s_and_b32 s24, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s25, s18, 16
-; SI-NEXT: s_and_b32 s26, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s27, s17, 16
-; SI-NEXT: s_and_b32 s28, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s29, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s25, s23, 16
+; SI-NEXT: s_lshr_b32 s24, s21, 16
+; SI-NEXT: s_lshr_b32 s15, s19, 16
+; SI-NEXT: s_lshr_b32 s14, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB45_3
; SI-NEXT: .LBB45_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr15
; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
; SI-NEXT: .LBB45_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB45_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
-; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
+; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
+; SI-NEXT: v_lshr_b64 v[8:9], v[6:7], 16
+; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
+; SI-NEXT: v_lshr_b64 v[9:10], v[4:5], 16
+; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
+; SI-NEXT: v_lshr_b64 v[10:11], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; SI-NEXT: s_branch .LBB45_6
; SI-NEXT: .LBB45_5:
-; SI-NEXT: v_mov_b32_e32 v0, s29
-; SI-NEXT: v_mov_b32_e32 v1, s28
-; SI-NEXT: v_mov_b32_e32 v2, s27
-; SI-NEXT: v_mov_b32_e32 v15, s26
-; SI-NEXT: v_mov_b32_e32 v3, s25
-; SI-NEXT: v_mov_b32_e32 v14, s24
-; SI-NEXT: v_mov_b32_e32 v4, s15
-; SI-NEXT: v_mov_b32_e32 v13, s14
-; SI-NEXT: v_mov_b32_e32 v5, s13
-; SI-NEXT: v_mov_b32_e32 v12, s12
-; SI-NEXT: v_mov_b32_e32 v6, s11
-; SI-NEXT: v_mov_b32_e32 v11, s10
-; SI-NEXT: v_mov_b32_e32 v7, s9
-; SI-NEXT: v_mov_b32_e32 v10, s8
-; SI-NEXT: v_mov_b32_e32 v8, s7
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v15, s14
+; SI-NEXT: v_mov_b32_e32 v14, s15
+; SI-NEXT: v_mov_b32_e32 v13, s24
+; SI-NEXT: v_mov_b32_e32 v12, s25
+; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: v_mov_b32_e32 v9, s6
+; SI-NEXT: v_mov_b32_e32 v10, s8
+; SI-NEXT: v_mov_b32_e32 v11, s10
; SI-NEXT: .LBB45_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[15:16], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[14:15], 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[13:14], 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v5
-; SI-NEXT: v_lshr_b64 v[4:5], v[12:13], 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[11:12], 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[10:11], 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[8:9], 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v15
+; SI-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; SI-NEXT: v_or_b32_e32 v4, v4, v9
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; SI-NEXT: v_or_b32_e32 v1, v1, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8f32_to_v16bf16_scalar:
@@ -11084,39 +10924,31 @@ define <8 x float> @bitcast_v16bf16_to_v8f32(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v8f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v16, v7
+; SI-NEXT: v_mov_b32_e32 v9, v6
+; SI-NEXT: v_mov_b32_e32 v10, v5
+; SI-NEXT: v_mov_b32_e32 v11, v4
+; SI-NEXT: v_mov_b32_e32 v12, v3
+; SI-NEXT: v_mov_b32_e32 v13, v2
+; SI-NEXT: v_mov_b32_e32 v14, v1
+; SI-NEXT: v_mov_b32_e32 v15, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v15
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v7
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -11128,85 +10960,77 @@ define <8 x float> @bitcast_v16bf16_to_v8f32(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB46_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v23, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v21, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v19, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v17, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v15, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v13, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v11, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v9, 16
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v22
+; SI-NEXT: v_or_b32_e32 v2, v2, v21
+; SI-NEXT: v_or_b32_e32 v3, v3, v20
+; SI-NEXT: v_or_b32_e32 v4, v4, v19
+; SI-NEXT: v_or_b32_e32 v5, v5, v18
+; SI-NEXT: v_or_b32_e32 v6, v6, v17
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB46_2
; SI-NEXT: .LBB46_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v22
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v21
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v20
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v14
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v19
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v18
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v10
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v17
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -11830,117 +11654,111 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; SI-LABEL: bitcast_v16bf16_to_v8f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s22, 16
+; SI-NEXT: s_lshr_b32 s25, s21, 16
+; SI-NEXT: s_lshr_b32 s26, s20, 16
+; SI-NEXT: s_lshr_b32 s27, s19, 16
+; SI-NEXT: s_lshr_b32 s28, s18, 16
+; SI-NEXT: s_lshr_b32 s29, s17, 16
+; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
-; SI-NEXT: v_lshr_b64 v[0:1], v[23:24], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[21:22], 16
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v30
-; SI-NEXT: v_lshr_b64 v[2:3], v[19:20], 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v29
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[15:16], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; SI-NEXT: v_lshr_b64 v[5:6], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v26
-; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
-; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s40, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s29, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s28, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s27, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_and_b32 s8, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s26, 16
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s25, 16
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s10, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s15, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB47_3
; SI-NEXT: .LBB47_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
; SI-NEXT: .LBB47_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
; SI-NEXT: s_cbranch_scc1 .LBB47_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v8, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB47_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16bf16_to_v8f32_scalar:
@@ -17645,51 +17463,26 @@ define <16 x bfloat> @bitcast_v4i64_to_v16bf16(<4 x i64> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB64_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: .LBB64_2: ; %Flow
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB64_4
; SI-NEXT: ; %bb.3: ; %cmp.true
@@ -17701,56 +17494,40 @@ define <16 x bfloat> @bitcast_v4i64_to_v16bf16(<4 x i64> %a, i32 %b) #0 {
; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; SI-NEXT: .LBB64_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v14
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v12
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v15
+; SI-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; SI-NEXT: v_or_b32_e32 v4, v4, v9
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v13
+; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i64_to_v16bf16:
@@ -17840,105 +17617,81 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s23, 16
-; SI-NEXT: s_and_b32 s9, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s22, 16
-; SI-NEXT: s_and_b32 s11, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s21, 16
-; SI-NEXT: s_and_b32 s13, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s20, 16
-; SI-NEXT: s_and_b32 s15, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s19, 16
-; SI-NEXT: s_and_b32 s25, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s24, s18, 16
-; SI-NEXT: s_and_b32 s27, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s26, s17, 16
-; SI-NEXT: s_and_b32 s29, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s28, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s21, 16
+; SI-NEXT: s_lshr_b32 s24, s19, 16
+; SI-NEXT: s_lshr_b32 s25, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB65_3
; SI-NEXT: .LBB65_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr25
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr24
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: .LBB65_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB65_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_u32 s4, s16, 3
-; SI-NEXT: s_addc_u32 s5, s17, 0
-; SI-NEXT: s_add_u32 s16, s18, 3
-; SI-NEXT: s_addc_u32 s14, s19, 0
-; SI-NEXT: s_add_u32 s12, s20, 3
-; SI-NEXT: s_addc_u32 s10, s21, 0
-; SI-NEXT: s_add_u32 s8, s22, 3
-; SI-NEXT: s_addc_u32 s6, s23, 0
-; SI-NEXT: s_and_b32 s7, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s9, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_and_b32 s11, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s13, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s15, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_and_b32 s25, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s24, s16, 16
-; SI-NEXT: s_and_b32 s27, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s26, s5, 16
-; SI-NEXT: s_and_b32 s29, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s28, s4, 16
+; SI-NEXT: s_add_u32 s22, s22, 3
+; SI-NEXT: s_addc_u32 s23, s23, 0
+; SI-NEXT: s_add_u32 s20, s20, 3
+; SI-NEXT: s_addc_u32 s21, s21, 0
+; SI-NEXT: s_add_u32 s18, s18, 3
+; SI-NEXT: s_addc_u32 s19, s19, 0
+; SI-NEXT: s_add_u32 s16, s16, 3
+; SI-NEXT: s_addc_u32 s17, s17, 0
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s21, 16
+; SI-NEXT: s_lshr_b32 s24, s19, 16
+; SI-NEXT: s_lshr_b32 s25, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
; SI-NEXT: .LBB65_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s29
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s27
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s26
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s25
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s24
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_and_b32 s7, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s25, 16
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_and_b32 s9, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: s_and_b32 s9, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s24, 16
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s10, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s6, s10, s6
+; SI-NEXT: s_and_b32 s10, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s15, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s11, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s11, s4
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: v_mov_b32_e32 v4, s6
+; SI-NEXT: v_mov_b32_e32 v5, s10
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i64_to_v16bf16_scalar:
@@ -18062,39 +17815,31 @@ define <4 x i64> @bitcast_v16bf16_to_v4i64(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v4i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v16, v7
+; SI-NEXT: v_mov_b32_e32 v9, v6
+; SI-NEXT: v_mov_b32_e32 v10, v5
+; SI-NEXT: v_mov_b32_e32 v11, v4
+; SI-NEXT: v_mov_b32_e32 v12, v3
+; SI-NEXT: v_mov_b32_e32 v13, v2
+; SI-NEXT: v_mov_b32_e32 v14, v1
+; SI-NEXT: v_mov_b32_e32 v15, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v15
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v7
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -18106,85 +17851,77 @@ define <4 x i64> @bitcast_v16bf16_to_v4i64(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB66_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v23, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v21, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v19, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v17, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v15, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v13, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v11, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v9, 16
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v22
+; SI-NEXT: v_or_b32_e32 v2, v2, v21
+; SI-NEXT: v_or_b32_e32 v3, v3, v20
+; SI-NEXT: v_or_b32_e32 v4, v4, v19
+; SI-NEXT: v_or_b32_e32 v5, v5, v18
+; SI-NEXT: v_or_b32_e32 v6, v6, v17
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB66_2
; SI-NEXT: .LBB66_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v22
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v21
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v20
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v14
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v19
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v18
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v10
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v17
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -18808,117 +18545,111 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; SI-LABEL: bitcast_v16bf16_to_v4i64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s22, 16
+; SI-NEXT: s_lshr_b32 s25, s21, 16
+; SI-NEXT: s_lshr_b32 s26, s20, 16
+; SI-NEXT: s_lshr_b32 s27, s19, 16
+; SI-NEXT: s_lshr_b32 s28, s18, 16
+; SI-NEXT: s_lshr_b32 s29, s17, 16
+; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
-; SI-NEXT: v_lshr_b64 v[0:1], v[23:24], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[21:22], 16
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v30
-; SI-NEXT: v_lshr_b64 v[2:3], v[19:20], 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v29
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[15:16], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; SI-NEXT: v_lshr_b64 v[5:6], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v26
-; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
-; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s40, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s29, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s28, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s27, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_and_b32 s8, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s26, 16
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s25, 16
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s10, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s15, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB67_3
; SI-NEXT: .LBB67_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
; SI-NEXT: .LBB67_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
; SI-NEXT: s_cbranch_scc1 .LBB67_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v8, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB67_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB67_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16bf16_to_v4i64_scalar:
@@ -24072,47 +23803,26 @@ define <16 x bfloat> @bitcast_v4f64_to_v16bf16(<4 x double> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB80_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: .LBB80_2: ; %Flow
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB80_4
; SI-NEXT: ; %bb.3: ; %cmp.true
@@ -24120,56 +23830,40 @@ define <16 x bfloat> @bitcast_v4f64_to_v16bf16(<4 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[2:3], v[2:3], 1.0
; SI-NEXT: v_add_f64 v[6:7], v[6:7], 1.0
; SI-NEXT: v_add_f64 v[4:5], v[4:5], 1.0
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; SI-NEXT: v_alignbit_b32 v8, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v9, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v10, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v13, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; SI-NEXT: .LBB80_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v14
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v12
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v13
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v15
+; SI-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; SI-NEXT: v_or_b32_e32 v4, v4, v9
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; SI-NEXT: v_or_b32_e32 v1, v1, v13
+; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4f64_to_v16bf16:
@@ -24247,119 +23941,87 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s29, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s28, s23, 16
-; SI-NEXT: s_and_b32 s27, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s26, s22, 16
-; SI-NEXT: s_and_b32 s25, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s24, s21, 16
-; SI-NEXT: s_and_b32 s15, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s20, 16
-; SI-NEXT: s_and_b32 s13, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s19, 16
-; SI-NEXT: s_and_b32 s11, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s18, 16
-; SI-NEXT: s_and_b32 s9, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s17, 16
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s25, s23, 16
+; SI-NEXT: s_lshr_b32 s24, s21, 16
+; SI-NEXT: s_lshr_b32 s15, s19, 16
+; SI-NEXT: s_lshr_b32 s14, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB81_3
; SI-NEXT: .LBB81_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
; SI-NEXT: .LBB81_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB81_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
+; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; SI-NEXT: v_add_f64 v[4:5], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[13:14], s[20:21], 1.0
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v5
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v14
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshr_b64 v[8:9], v[6:7], 16
+; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; SI-NEXT: v_lshr_b64 v[9:10], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[10:11], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
; SI-NEXT: s_branch .LBB81_6
; SI-NEXT: .LBB81_5:
-; SI-NEXT: v_mov_b32_e32 v9, s29
-; SI-NEXT: v_mov_b32_e32 v8, s28
-; SI-NEXT: v_mov_b32_e32 v10, s27
-; SI-NEXT: v_mov_b32_e32 v7, s26
-; SI-NEXT: v_mov_b32_e32 v11, s25
-; SI-NEXT: v_mov_b32_e32 v6, s24
-; SI-NEXT: v_mov_b32_e32 v12, s15
-; SI-NEXT: v_mov_b32_e32 v5, s14
-; SI-NEXT: v_mov_b32_e32 v13, s13
-; SI-NEXT: v_mov_b32_e32 v4, s12
-; SI-NEXT: v_mov_b32_e32 v14, s11
-; SI-NEXT: v_mov_b32_e32 v3, s10
-; SI-NEXT: v_mov_b32_e32 v15, s9
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v13, s25
+; SI-NEXT: v_mov_b32_e32 v14, s24
+; SI-NEXT: v_mov_b32_e32 v15, s15
+; SI-NEXT: v_mov_b32_e32 v12, s14
+; SI-NEXT: v_mov_b32_e32 v11, s10
+; SI-NEXT: v_mov_b32_e32 v10, s8
+; SI-NEXT: v_mov_b32_e32 v9, s6
+; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: .LBB81_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[15:16], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[14:15], 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[13:14], 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v5
-; SI-NEXT: v_lshr_b64 v[4:5], v[12:13], 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[11:12], 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[10:11], 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[8:9], 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; SI-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v15
+; SI-NEXT: v_or_b32_e32 v4, v4, v9
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v14
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; SI-NEXT: v_or_b32_e32 v1, v1, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4f64_to_v16bf16_scalar:
@@ -24477,39 +24139,31 @@ define <4 x double> @bitcast_v16bf16_to_v4f64(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v4f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v16, v7
+; SI-NEXT: v_mov_b32_e32 v9, v6
+; SI-NEXT: v_mov_b32_e32 v10, v5
+; SI-NEXT: v_mov_b32_e32 v11, v4
+; SI-NEXT: v_mov_b32_e32 v12, v3
+; SI-NEXT: v_mov_b32_e32 v13, v2
+; SI-NEXT: v_mov_b32_e32 v14, v1
+; SI-NEXT: v_mov_b32_e32 v15, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v15
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v7
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -24521,85 +24175,77 @@ define <4 x double> @bitcast_v16bf16_to_v4f64(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB82_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v23, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v21, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v19, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v17, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v15, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v13, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v11, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v9, 16
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v22
+; SI-NEXT: v_or_b32_e32 v2, v2, v21
+; SI-NEXT: v_or_b32_e32 v3, v3, v20
+; SI-NEXT: v_or_b32_e32 v4, v4, v19
+; SI-NEXT: v_or_b32_e32 v5, v5, v18
+; SI-NEXT: v_or_b32_e32 v6, v6, v17
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB82_2
; SI-NEXT: .LBB82_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v22
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v21
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v20
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v14
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v19
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v18
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v10
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v17
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
@@ -25223,117 +24869,111 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; SI-LABEL: bitcast_v16bf16_to_v4f64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s14, s23, 16
+; SI-NEXT: s_lshr_b32 s15, s22, 16
+; SI-NEXT: s_lshr_b32 s25, s21, 16
+; SI-NEXT: s_lshr_b32 s26, s20, 16
+; SI-NEXT: s_lshr_b32 s27, s19, 16
+; SI-NEXT: s_lshr_b32 s28, s18, 16
+; SI-NEXT: s_lshr_b32 s29, s17, 16
+; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
-; SI-NEXT: v_lshr_b64 v[0:1], v[23:24], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[21:22], 16
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v30
-; SI-NEXT: v_lshr_b64 v[2:3], v[19:20], 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v29
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[15:16], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; SI-NEXT: v_lshr_b64 v[5:6], v[13:14], 16
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v26
-; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
-; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s40, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s29, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s28, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s27, 16
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_and_b32 s8, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s26, 16
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s25, 16
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_and_b32 s10, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s15, 16
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB83_3
; SI-NEXT: .LBB83_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
; SI-NEXT: .LBB83_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
; SI-NEXT: s_cbranch_scc1 .LBB83_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v8, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v25
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v8
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB83_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB83_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v5, s9
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16bf16_to_v4f64_scalar:
@@ -29895,42 +29535,59 @@ define <16 x bfloat> @bitcast_v16i16_to_v16bf16(<16 x i16> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16i16_to_v16bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v4
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v12
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB92_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v17, v1, v25
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v15, v1, v27
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_or_b32_e32 v22, v0, v24
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v13, v1, v29
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_or_b32_e32 v18, v0, v26
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v8, v1, v31
+; SI-NEXT: v_or_b32_e32 v16, v0, v28
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_alignbit_b32 v19, v17, v24, 16
+; SI-NEXT: v_alignbit_b32 v20, v15, v26, 16
+; SI-NEXT: v_alignbit_b32 v21, v13, v28, 16
+; SI-NEXT: v_alignbit_b32 v23, v8, v30, 16
+; SI-NEXT: v_or_b32_e32 v12, v0, v30
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
@@ -29939,93 +29596,85 @@ define <16 x bfloat> @bitcast_v16i16_to_v16bf16(<16 x i16> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; %bb.2: ; %Flow
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: .LBB92_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB92_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v7, v15, v7
+; SI-NEXT: v_or_b32_e32 v6, v30, v6
; SI-NEXT: s_mov_b32 s6, 0x30000
-; SI-NEXT: v_or_b32_e32 v6, v14, v6
-; SI-NEXT: v_or_b32_e32 v5, v13, v5
-; SI-NEXT: v_or_b32_e32 v4, v12, v4
-; SI-NEXT: v_or_b32_e32 v3, v11, v3
-; SI-NEXT: v_or_b32_e32 v2, v10, v2
-; SI-NEXT: v_or_b32_e32 v1, v9, v1
-; SI-NEXT: v_or_b32_e32 v0, v8, v0
-; SI-NEXT: v_add_i32_e32 v7, vcc, 0x30000, v7
-; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v4
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v3
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v5
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v6
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; SI-NEXT: v_or_b32_e32 v4, v28, v4
+; SI-NEXT: v_or_b32_e32 v2, v26, v2
+; SI-NEXT: v_or_b32_e32 v0, v24, v0
+; SI-NEXT: v_add_i32_e32 v12, vcc, 0x30000, v6
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v7
+; SI-NEXT: v_add_i32_e32 v16, vcc, s6, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v5
+; SI-NEXT: v_add_i32_e32 v18, vcc, s6, v2
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v3
+; SI-NEXT: v_add_i32_e32 v22, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v6, v31, v6
+; SI-NEXT: v_or_b32_e32 v4, v29, v4
+; SI-NEXT: v_or_b32_e32 v2, v27, v2
+; SI-NEXT: v_or_b32_e32 v0, v25, v0
+; SI-NEXT: v_add_i32_e32 v8, vcc, s6, v6
+; SI-NEXT: v_add_i32_e32 v13, vcc, s6, v4
+; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v2
+; SI-NEXT: v_add_i32_e32 v17, vcc, s6, v0
+; SI-NEXT: v_alignbit_b32 v19, v17, v22, 16
+; SI-NEXT: v_alignbit_b32 v20, v15, v18, 16
+; SI-NEXT: v_alignbit_b32 v21, v13, v16, 16
+; SI-NEXT: v_alignbit_b32 v23, v8, v12, 16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
; SI-NEXT: .LBB92_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v17
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v19
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v12
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v21
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v23
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v19
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v20
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v21
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i16_to_v16bf16:
@@ -30128,89 +29777,97 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; SI-LABEL: bitcast_v16i16_to_v16bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s47, s23, 16
-; SI-NEXT: s_lshr_b32 s46, s22, 16
-; SI-NEXT: s_lshr_b32 s45, s21, 16
-; SI-NEXT: s_lshr_b32 s44, s20, 16
-; SI-NEXT: s_lshr_b32 s43, s19, 16
-; SI-NEXT: s_lshr_b32 s42, s18, 16
-; SI-NEXT: s_lshr_b32 s41, s17, 16
-; SI-NEXT: s_lshr_b32 s40, s16, 16
+; SI-NEXT: s_lshr_b32 s43, s23, 16
+; SI-NEXT: s_lshr_b32 s47, s22, 16
+; SI-NEXT: s_lshr_b32 s42, s21, 16
+; SI-NEXT: s_lshr_b32 s46, s20, 16
+; SI-NEXT: s_lshr_b32 s41, s19, 16
+; SI-NEXT: s_lshr_b32 s45, s18, 16
+; SI-NEXT: s_lshr_b32 s40, s17, 16
+; SI-NEXT: s_lshr_b32 s44, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s25, s16, 16
-; SI-NEXT: s_lshl_b32 s29, s40, 16
-; SI-NEXT: s_lshl_b32 s15, s17, 16
-; SI-NEXT: s_lshl_b32 s28, s41, 16
-; SI-NEXT: s_lshl_b32 s13, s18, 16
-; SI-NEXT: s_lshl_b32 s27, s42, 16
-; SI-NEXT: s_lshl_b32 s11, s19, 16
-; SI-NEXT: s_lshl_b32 s26, s43, 16
-; SI-NEXT: s_lshl_b32 s10, s20, 16
-; SI-NEXT: s_lshl_b32 s24, s44, 16
-; SI-NEXT: s_lshl_b32 s8, s21, 16
-; SI-NEXT: s_lshl_b32 s14, s45, 16
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_lshl_b32 s12, s46, 16
-; SI-NEXT: s_lshl_b32 s6, s23, 16
-; SI-NEXT: s_lshl_b32 s9, s47, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s40, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s44, 16
+; SI-NEXT: s_or_b32 s57, s5, s7
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s41, 16
+; SI-NEXT: s_or_b32 s10, s4, s56
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s58, s45, 16
+; SI-NEXT: s_or_b32 s59, s5, s7
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s42, 16
+; SI-NEXT: s_or_b32 s8, s4, s58
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s60, s46, 16
+; SI-NEXT: s_or_b32 s61, s5, s7
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s43, 16
+; SI-NEXT: s_or_b32 s6, s4, s60
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s62, s47, 16
+; SI-NEXT: s_or_b32 s63, s5, s7
+; SI-NEXT: s_or_b32 s4, s4, s62
+; SI-NEXT: s_lshr_b64 s[12:13], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[62:63], 16
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_mov_b32 s11, s57
+; SI-NEXT: s_mov_b32 s9, s59
+; SI-NEXT: s_mov_b32 s7, s61
+; SI-NEXT: s_mov_b32 s5, s63
; SI-NEXT: s_branch .LBB93_3
; SI-NEXT: .LBB93_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: s_mov_b64 s[28:29], -1
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr26
; SI-NEXT: .LBB93_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
; SI-NEXT: s_cbranch_scc1 .LBB93_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s47, 16
; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s47, 16
+; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s46, 16
-; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s45, 16
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s43, 16
; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s46, 16
+; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s44, 16
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s43, 16
+; SI-NEXT: s_and_b32 s7, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s42, 16
; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s8, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s45, 16
+; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s42, 16
-; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s41, 16
+; SI-NEXT: s_and_b32 s9, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s41, 16
; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_and_b32 s10, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s44, 16
+; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s16, 0xffff
+; SI-NEXT: s_and_b32 s11, s17, 0xffff
; SI-NEXT: s_lshl_b32 s12, s40, 16
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: s_add_i32 s4, s4, 0x30000
@@ -30221,55 +29878,47 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: s_and_b32 s29, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s25, s11, 16
-; SI-NEXT: s_and_b32 s28, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s10, 16
-; SI-NEXT: s_and_b32 s27, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s9, 16
-; SI-NEXT: s_and_b32 s26, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s8, 16
-; SI-NEXT: s_and_b32 s24, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s7, 16
-; SI-NEXT: s_and_b32 s14, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s6, 16
-; SI-NEXT: s_and_b32 s12, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s5, 16
-; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s40, s11, 16
+; SI-NEXT: s_lshr_b32 s41, s9, 16
+; SI-NEXT: s_lshr_b32 s42, s7, 16
+; SI-NEXT: s_lshr_b32 s43, s5, 16
; SI-NEXT: .LBB93_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s29
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s25
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s28
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s15
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s27
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s13
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s26
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s11
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s24
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s14
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s12
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s10, s10, s12
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s40, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s8, s8, s12
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s41, 16
+; SI-NEXT: s_or_b32 s9, s9, s12
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s24, 16
+; SI-NEXT: s_or_b32 s6, s6, s12
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s42, 16
+; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s26, 16
+; SI-NEXT: s_or_b32 s4, s4, s12
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s43, 16
+; SI-NEXT: s_or_b32 s5, s5, s12
+; SI-NEXT: v_mov_b32_e32 v0, s10
+; SI-NEXT: v_mov_b32_e32 v1, s11
+; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: v_mov_b32_e32 v4, s6
+; SI-NEXT: v_mov_b32_e32 v5, s7
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_mov_b32_e32 v7, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i16_to_v16bf16_scalar:
@@ -30430,176 +30079,101 @@ define <16 x i16> @bitcast_v16bf16_to_v16i16(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v16i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v9
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB94_2
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v31
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v22
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: .LBB94_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB94_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v1
+; SI-NEXT: s_cbranch_execz .LBB94_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v28
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v26
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v24
-; SI-NEXT: v_alignbit_b32 v4, v4, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v22
-; SI-NEXT: v_alignbit_b32 v6, v6, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_alignbit_b32 v0, v16, v0, 16
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_alignbit_b32 v2, v16, v2, 16
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v20
-; SI-NEXT: v_alignbit_b32 v7, v8, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_alignbit_b32 v5, v9, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v15
-; SI-NEXT: v_alignbit_b32 v3, v10, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v1, v11, v1, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_alignbit_b32 v12, v1, v12, 16
-; SI-NEXT: v_alignbit_b32 v13, v3, v13, 16
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_alignbit_b32 v4, v16, v4, 16
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v13
+; SI-NEXT: v_alignbit_b32 v7, v9, v7, 16
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_alignbit_b32 v5, v10, v5, 16
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_alignbit_b32 v3, v11, v3, 16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_alignbit_b32 v1, v12, v1, 16
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_alignbit_b32 v6, v16, v6, 16
+; SI-NEXT: v_alignbit_b32 v16, v1, v8, 16
+; SI-NEXT: v_alignbit_b32 v15, v3, v15, 16
; SI-NEXT: v_alignbit_b32 v14, v5, v14, 16
-; SI-NEXT: v_alignbit_b32 v17, v7, v17, 16
-; SI-NEXT: .LBB94_4: ; %end
+; SI-NEXT: v_alignbit_b32 v13, v7, v13, 16
+; SI-NEXT: .LBB94_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_or_b32_e32 v1, v1, v11
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v8
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; SI-NEXT: v_or_b32_e32 v1, v1, v8
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v13
-; SI-NEXT: v_or_b32_e32 v3, v3, v10
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; SI-NEXT: v_or_b32_e32 v2, v2, v8
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v8
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; SI-NEXT: v_or_b32_e32 v5, v5, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; SI-NEXT: v_or_b32_e32 v4, v4, v8
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v8
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_or_b32_e32 v0, v0, v12
-; SI-NEXT: v_or_b32_e32 v2, v2, v11
-; SI-NEXT: v_or_b32_e32 v4, v4, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -31200,163 +30774,126 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; SI-LABEL: bitcast_v16bf16_to_v16i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s23, 16
+; SI-NEXT: s_lshr_b32 s9, s22, 16
+; SI-NEXT: s_lshr_b32 s8, s21, 16
+; SI-NEXT: s_lshr_b32 s11, s20, 16
+; SI-NEXT: s_lshr_b32 s7, s19, 16
+; SI-NEXT: s_lshr_b32 s12, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v31
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v2
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
; SI-NEXT: s_branch .LBB95_3
; SI-NEXT: .LBB95_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: .LBB95_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB95_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v11
-; SI-NEXT: v_lshr_b64 v[13:14], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v9
-; SI-NEXT: v_lshr_b64 v[14:15], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v6
-; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v8
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v9
-; SI-NEXT: v_lshr_b64 v[9:10], v[17:18], 16
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v19
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v10
-; SI-NEXT: v_lshr_b64 v[11:12], v[21:22], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[10:11], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[8:9], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[6:7], 16
-; SI-NEXT: .LBB95_5: ; %end
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v23
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v22
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v24
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v20
-; SI-NEXT: v_or_b32_e32 v3, v3, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_lshr_b64 v[14:15], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v2
+; SI-NEXT: s_lshl_b32 s5, s23, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_add_f32_e32 v12, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s10, 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v1, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s21, 16
+; SI-NEXT: v_add_f32_e32 v16, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s19, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v18, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s6, v2
+; SI-NEXT: v_add_f32_e32 v22, s5, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_lshr_b64 v[7:8], v[12:13], 16
+; SI-NEXT: v_lshr_b64 v[20:21], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
+; SI-NEXT: v_lshr_b64 v[9:10], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[18:19], 16
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; SI-NEXT: v_lshr_b64 v[26:27], v[3:4], 16
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[6:7], 16
+; SI-NEXT: s_branch .LBB95_6
+; SI-NEXT: .LBB95_5:
+; SI-NEXT: v_mov_b32_e32 v23, s6
+; SI-NEXT: v_mov_b32_e32 v19, s7
+; SI-NEXT: v_mov_b32_e32 v17, s8
+; SI-NEXT: v_mov_b32_e32 v13, s10
+; SI-NEXT: v_mov_b32_e32 v20, s22
+; SI-NEXT: v_mov_b32_e32 v15, s20
+; SI-NEXT: v_mov_b32_e32 v14, s18
+; SI-NEXT: v_mov_b32_e32 v11, s16
+; SI-NEXT: v_mov_b32_e32 v26, s13
+; SI-NEXT: v_mov_b32_e32 v2, s12
+; SI-NEXT: v_mov_b32_e32 v24, s11
+; SI-NEXT: v_mov_b32_e32 v21, s9
+; SI-NEXT: v_mov_b32_e32 v4, s17
+; SI-NEXT: v_mov_b32_e32 v1, s19
+; SI-NEXT: v_mov_b32_e32 v9, s21
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: .LBB95_6: ; %end
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v26
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v23
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_or_b32_e32 v3, v4, v3
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v24
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_and_b32_e32 v5, 0xffff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v26
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v21
; SI-NEXT: v_or_b32_e32 v6, v6, v8
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v13
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -35238,51 +34775,59 @@ define <16 x bfloat> @bitcast_v16f16_to_v16bf16(<16 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16f16_to_v16bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v24
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v26
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v27
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v28
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v31
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; SI-NEXT: v_or_b32_e32 v18, v1, v8
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; SI-NEXT: v_or_b32_e32 v19, v3, v8
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v27
+; SI-NEXT: v_or_b32_e32 v17, v5, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v26
+; SI-NEXT: v_or_b32_e32 v16, v7, v8
+; SI-NEXT: v_or_b32_e32 v8, v0, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; SI-NEXT: v_or_b32_e32 v13, v0, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
+; SI-NEXT: v_or_b32_e32 v14, v0, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_alignbit_b32 v20, v18, v1, 16
+; SI-NEXT: v_alignbit_b32 v21, v19, v3, 16
+; SI-NEXT: v_alignbit_b32 v22, v17, v5, 16
+; SI-NEXT: v_alignbit_b32 v23, v16, v7, 16
+; SI-NEXT: v_or_b32_e32 v15, v0, v7
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
@@ -35291,116 +34836,108 @@ define <16 x bfloat> @bitcast_v16f16_to_v16bf16(<16 x half> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: .LBB100_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v31
-; SI-NEXT: v_cvt_f32_f16_e32 v9, v30
-; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v23, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v9
-; SI-NEXT: v_cvt_f16_f32_e32 v20, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v29
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v8, v27
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v26
; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v18, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v28
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v8
; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_or_b32_e32 v8, v0, v20
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v13
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v25
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v14
+; SI-NEXT: v_or_b32_e32 v14, v4, v22
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v9
+; SI-NEXT: v_or_b32_e32 v13, v2, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v24
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT: v_cvt_f16_f32_e32 v9, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v7
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v15, v4, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v9
+; SI-NEXT: v_or_b32_e32 v16, v6, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v10
; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v9, v24
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v27
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v11, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v12
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v26
+; SI-NEXT: v_cvt_f16_f32_e32 v10, v7
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v12, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v25
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v12, v4
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; SI-NEXT: v_or_b32_e32 v19, v3, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v12
+; SI-NEXT: v_or_b32_e32 v17, v5, v6
+; SI-NEXT: v_or_b32_e32 v18, v1, v3
+; SI-NEXT: v_alignbit_b32 v20, v18, v20, 16
+; SI-NEXT: v_alignbit_b32 v21, v19, v0, 16
+; SI-NEXT: v_alignbit_b32 v22, v17, v22, 16
+; SI-NEXT: v_alignbit_b32 v23, v16, v2, 16
; SI-NEXT: .LBB100_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v9
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v10
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v12
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v14
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v20
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v12
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v21
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v22
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f16_to_v16bf16:
@@ -35504,174 +35041,178 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; SI-LABEL: bitcast_v16f16_to_v16bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s14, s23, 16
-; SI-NEXT: s_lshr_b32 s12, s22, 16
-; SI-NEXT: s_lshr_b32 s11, s21, 16
-; SI-NEXT: s_lshr_b32 s10, s20, 16
-; SI-NEXT: s_lshr_b32 s9, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s7, s17, 16
-; SI-NEXT: s_lshr_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s43, s23, 16
+; SI-NEXT: s_lshr_b32 s44, s22, 16
+; SI-NEXT: s_lshr_b32 s42, s21, 16
+; SI-NEXT: s_lshr_b32 s45, s20, 16
+; SI-NEXT: s_lshr_b32 s41, s19, 16
+; SI-NEXT: s_lshr_b32 s46, s18, 16
+; SI-NEXT: s_lshr_b32 s40, s17, 16
+; SI-NEXT: s_lshr_b32 s56, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s13, s16, 16
-; SI-NEXT: s_lshl_b32 s15, s6, 16
-; SI-NEXT: s_lshl_b32 s24, s17, 16
-; SI-NEXT: s_lshl_b32 s25, s7, 16
-; SI-NEXT: s_lshl_b32 s26, s18, 16
-; SI-NEXT: s_lshl_b32 s27, s8, 16
-; SI-NEXT: s_lshl_b32 s28, s19, 16
-; SI-NEXT: s_lshl_b32 s29, s9, 16
-; SI-NEXT: s_lshl_b32 s40, s20, 16
-; SI-NEXT: s_lshl_b32 s41, s10, 16
-; SI-NEXT: s_lshl_b32 s42, s21, 16
-; SI-NEXT: s_lshl_b32 s43, s11, 16
-; SI-NEXT: s_lshl_b32 s44, s22, 16
-; SI-NEXT: s_lshl_b32 s45, s12, 16
-; SI-NEXT: s_lshl_b32 s46, s23, 16
-; SI-NEXT: s_lshl_b32 s47, s14, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s56, 16
+; SI-NEXT: s_or_b32 s47, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s46, 16
+; SI-NEXT: s_or_b32 s57, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s45, 16
+; SI-NEXT: s_or_b32 s58, s5, s10
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s44, 16
+; SI-NEXT: s_or_b32 s59, s5, s14
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s40, 16
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s41, 16
+; SI-NEXT: s_or_b32 s7, s4, s7
+; SI-NEXT: s_lshr_b64 s[12:13], s[6:7], 16
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s42, 16
+; SI-NEXT: s_or_b32 s11, s4, s6
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s43, 16
+; SI-NEXT: s_or_b32 s15, s4, s6
+; SI-NEXT: s_lshr_b64 s[24:25], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[14:15], 16
+; SI-NEXT: s_mov_b64 s[28:29], 0
; SI-NEXT: s_branch .LBB101_3
; SI-NEXT: .LBB101_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr15
; SI-NEXT: .LBB101_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB101_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s14
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s23
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s56
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s46
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v9, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s22
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s11
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s21
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v16, v1, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s45
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
+; SI-NEXT: v_or_b32_e32 v19, v3, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s44
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v11, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s10
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s20
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v4
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_or_b32_e32 v18, v5, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s22
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s43
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s23
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v12, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s9
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s19
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v17, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s42
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_or_b32_e32 v20, v1, v6
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v21, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v7
+; SI-NEXT: v_or_b32_e32 v7, v3, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s41
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v21
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s17
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v13, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s8
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s7
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
-; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_or_b32_e32 v5, v5, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s19
+; SI-NEXT: v_cvt_f16_f32_e32 v22, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s40
+; SI-NEXT: v_lshr_b64 v[10:11], v[4:5], 16
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v16, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v17, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v16
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v17
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_cvt_f16_f32_e32 v23, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v8
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; SI-NEXT: v_or_b32_e32 v3, v3, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v8
+; SI-NEXT: v_lshr_b64 v[14:15], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[12:13], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[6:7], 16
; SI-NEXT: s_branch .LBB101_6
; SI-NEXT: .LBB101_5:
-; SI-NEXT: v_mov_b32_e32 v9, s47
-; SI-NEXT: v_mov_b32_e32 v8, s46
-; SI-NEXT: v_mov_b32_e32 v10, s45
-; SI-NEXT: v_mov_b32_e32 v7, s44
-; SI-NEXT: v_mov_b32_e32 v11, s43
-; SI-NEXT: v_mov_b32_e32 v6, s42
-; SI-NEXT: v_mov_b32_e32 v12, s41
-; SI-NEXT: v_mov_b32_e32 v5, s40
-; SI-NEXT: v_mov_b32_e32 v13, s29
-; SI-NEXT: v_mov_b32_e32 v4, s28
-; SI-NEXT: v_mov_b32_e32 v14, s27
-; SI-NEXT: v_mov_b32_e32 v3, s26
-; SI-NEXT: v_mov_b32_e32 v15, s25
-; SI-NEXT: v_mov_b32_e32 v2, s24
-; SI-NEXT: v_mov_b32_e32 v1, s15
-; SI-NEXT: v_mov_b32_e32 v0, s13
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v21, s42
+; SI-NEXT: v_mov_b32_e32 v22, s41
+; SI-NEXT: v_mov_b32_e32 v23, s40
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v5, s11
+; SI-NEXT: v_mov_b32_e32 v7, s15
+; SI-NEXT: v_mov_b32_e32 v20, s59
+; SI-NEXT: v_mov_b32_e32 v18, s58
+; SI-NEXT: v_mov_b32_e32 v19, s57
+; SI-NEXT: v_mov_b32_e32 v16, s47
+; SI-NEXT: v_mov_b32_e32 v14, s8
+; SI-NEXT: v_mov_b32_e32 v12, s12
+; SI-NEXT: v_mov_b32_e32 v10, s24
+; SI-NEXT: v_mov_b32_e32 v8, s26
; SI-NEXT: .LBB101_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[15:16], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[14:15], 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[13:14], 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v4
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v5
-; SI-NEXT: v_lshr_b64 v[4:5], v[12:13], 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[11:12], 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[10:11], 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[8:9], 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v12
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v22
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; SI-NEXT: v_or_b32_e32 v4, v4, v6
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v21
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v17
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f16_to_v16bf16_scalar:
@@ -35835,185 +35376,153 @@ define <16 x half> @bitcast_v16bf16_to_v16f16(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v16f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v6
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v12
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
-; SI-NEXT: v_alignbit_b32 v11, v1, v17, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v18
-; SI-NEXT: v_alignbit_b32 v12, v11, v0, 16
-; SI-NEXT: v_alignbit_b32 v10, v3, v19, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v26
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v20
-; SI-NEXT: v_alignbit_b32 v13, v10, v0, 16
-; SI-NEXT: v_alignbit_b32 v6, v5, v21, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v22
-; SI-NEXT: v_alignbit_b32 v14, v6, v0, 16
-; SI-NEXT: v_alignbit_b32 v8, v7, v25, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
-; SI-NEXT: v_alignbit_b32 v15, v8, v0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v23
-; SI-NEXT: v_alignbit_b32 v0, v0, v31, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v30, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v29, 16
-; SI-NEXT: v_alignbit_b32 v9, v9, v28, 16
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v19, v1, v24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v18, v1, v25
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_or_b32_e32 v8, v0, v31
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v16, v1, v26
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_or_b32_e32 v12, v0, v30
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v15, v1, v28
+; SI-NEXT: v_or_b32_e32 v14, v0, v29
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_alignbit_b32 v20, v19, v31, 16
+; SI-NEXT: v_alignbit_b32 v21, v18, v30, 16
+; SI-NEXT: v_alignbit_b32 v22, v16, v29, 16
+; SI-NEXT: v_alignbit_b32 v23, v15, v27, 16
+; SI-NEXT: v_or_b32_e32 v17, v0, v27
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: .LBB102_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v31
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v26
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v11
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v24
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v23
-; SI-NEXT: v_alignbit_b32 v4, v4, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_alignbit_b32 v9, v6, v1, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v20
-; SI-NEXT: v_alignbit_b32 v8, v7, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v20
+; SI-NEXT: v_alignbit_b32 v8, v8, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v30
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; SI-NEXT: v_alignbit_b32 v12, v9, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v29
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; SI-NEXT: v_alignbit_b32 v14, v9, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v27
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; SI-NEXT: v_alignbit_b32 v17, v9, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v28
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; SI-NEXT: v_alignbit_b32 v15, v9, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_alignbit_b32 v6, v5, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; SI-NEXT: v_alignbit_b32 v16, v10, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v25
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; SI-NEXT: v_alignbit_b32 v18, v11, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v24
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v10, v3, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v11, v1, v11, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_alignbit_b32 v12, v11, v12, 16
-; SI-NEXT: v_alignbit_b32 v13, v10, v13, 16
-; SI-NEXT: v_alignbit_b32 v14, v6, v14, 16
-; SI-NEXT: v_alignbit_b32 v15, v8, v15, 16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT: v_alignbit_b32 v19, v13, v1, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v20
+; SI-NEXT: v_alignbit_b32 v20, v19, v1, 16
+; SI-NEXT: v_alignbit_b32 v21, v18, v2, 16
+; SI-NEXT: v_alignbit_b32 v22, v16, v4, 16
+; SI-NEXT: v_alignbit_b32 v23, v15, v0, 16
; SI-NEXT: .LBB102_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_or_b32_e32 v1, v11, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v13
-; SI-NEXT: v_or_b32_e32 v3, v10, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v15
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_or_b32_e32 v0, v0, v12
-; SI-NEXT: v_or_b32_e32 v2, v2, v11
-; SI-NEXT: v_or_b32_e32 v4, v4, v10
-; SI-NEXT: v_or_b32_e32 v6, v6, v9
-; SI-NEXT: v_or_b32_e32 v7, v8, v7
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v20
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v21
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v22
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16bf16_to_v16f16:
@@ -36628,167 +36137,166 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; SI-LABEL: bitcast_v16bf16_to_v16f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s43, s23, 16
+; SI-NEXT: s_lshr_b32 s44, s22, 16
+; SI-NEXT: s_lshr_b32 s42, s21, 16
+; SI-NEXT: s_lshr_b32 s45, s20, 16
+; SI-NEXT: s_lshr_b32 s41, s19, 16
+; SI-NEXT: s_lshr_b32 s46, s18, 16
+; SI-NEXT: s_lshr_b32 s40, s17, 16
+; SI-NEXT: s_lshr_b32 s47, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s7
; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v36
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v39
-; SI-NEXT: v_lshr_b64 v[13:14], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
-; SI-NEXT: v_lshr_b64 v[10:11], v[17:18], 16
-; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v50
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v51
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v49
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v48
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v48
-; SI-NEXT: v_lshr_b64 v[33:34], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[21:22], v[29:30], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshr_b64 v[11:12], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[5:6], v[31:32], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[27:28], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[24:25], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s47, 16
+; SI-NEXT: s_or_b32 s56, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s46, 16
+; SI-NEXT: s_or_b32 s57, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s45, 16
+; SI-NEXT: s_or_b32 s58, s5, s8
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s44, 16
+; SI-NEXT: s_or_b32 s59, s5, s10
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s40, 16
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_lshr_b64 s[12:13], s[4:5], 16
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s41, 16
+; SI-NEXT: s_or_b32 s7, s4, s7
+; SI-NEXT: s_lshr_b64 s[14:15], s[6:7], 16
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s42, 16
+; SI-NEXT: s_or_b32 s9, s4, s6
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s43, 16
+; SI-NEXT: s_or_b32 s11, s4, s6
+; SI-NEXT: s_lshr_b64 s[24:25], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[10:11], 16
+; SI-NEXT: s_mov_b64 s[28:29], 0
; SI-NEXT: s_branch .LBB103_3
; SI-NEXT: .LBB103_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr11
; SI-NEXT: .LBB103_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB103_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v51
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; SI-NEXT: v_lshr_b64 v[5:6], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_lshr_b64 v[21:22], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v49
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
-; SI-NEXT: v_lshr_b64 v[22:23], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshr_b64 v[25:26], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[17:18], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_lshr_b64 v[33:34], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[6:7], 16
-; SI-NEXT: .LBB103_5: ; %end
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v4
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v21
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v33
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v20
-; SI-NEXT: v_or_b32_e32 v3, v3, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v22
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s47, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s46, 16
+; SI-NEXT: v_add_f32_e32 v24, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_lshr_b64 v[14:15], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s45, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v2
+; SI-NEXT: s_lshl_b32 s5, s23, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_add_f32_e32 v12, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s43, 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v1, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s21, 16
+; SI-NEXT: v_add_f32_e32 v16, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s42, 16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s19, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v18, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s41, 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s44, 16
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s5, v2
+; SI-NEXT: s_lshl_b32 s5, s17, 16
+; SI-NEXT: s_lshl_b32 s6, s40, 16
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, s6, v2
+; SI-NEXT: v_add_f32_e32 v22, s5, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_lshr_b64 v[7:8], v[12:13], 16
+; SI-NEXT: v_lshr_b64 v[20:21], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
+; SI-NEXT: v_lshr_b64 v[9:10], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[18:19], 16
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; SI-NEXT: v_lshr_b64 v[26:27], v[3:4], 16
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[6:7], 16
+; SI-NEXT: s_branch .LBB103_6
+; SI-NEXT: .LBB103_5:
+; SI-NEXT: v_mov_b32_e32 v23, s40
+; SI-NEXT: v_mov_b32_e32 v19, s41
+; SI-NEXT: v_mov_b32_e32 v17, s42
+; SI-NEXT: v_mov_b32_e32 v13, s43
+; SI-NEXT: v_mov_b32_e32 v20, s59
+; SI-NEXT: v_mov_b32_e32 v15, s58
+; SI-NEXT: v_mov_b32_e32 v14, s57
+; SI-NEXT: v_mov_b32_e32 v11, s56
+; SI-NEXT: v_mov_b32_e32 v26, s12
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: v_mov_b32_e32 v24, s24
+; SI-NEXT: v_mov_b32_e32 v21, s26
+; SI-NEXT: v_mov_b32_e32 v4, s5
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: v_mov_b32_e32 v9, s9
+; SI-NEXT: v_mov_b32_e32 v7, s11
+; SI-NEXT: .LBB103_6: ; %end
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v26
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v23
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_or_b32_e32 v3, v4, v3
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v24
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v18
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v34
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v21
; SI-NEXT: v_or_b32_e32 v6, v6, v8
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v13
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -40748,46 +40256,37 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v16bf16_to_v32i8:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v36, v7
+; SI-NEXT: v_mov_b32_e32 v39, v6
+; SI-NEXT: v_mov_b32_e32 v34, v5
+; SI-NEXT: v_mov_b32_e32 v38, v4
+; SI-NEXT: v_mov_b32_e32 v33, v3
+; SI-NEXT: v_mov_b32_e32 v37, v2
+; SI-NEXT: v_mov_b32_e32 v32, v1
+; SI-NEXT: v_mov_b32_e32 v35, v0
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v36
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v39
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v34
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v38
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v33
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v37
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v35
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v7
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v30
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: ; implicit-def: $vgpr9
@@ -40795,7 +40294,6 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr17
@@ -40803,7 +40301,6 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $vgpr25
@@ -40811,7 +40308,6 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr28
; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -40823,22 +40319,22 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB108_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v39
-; SI-NEXT: v_alignbit_b32 v8, v5, v48, 16
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v35
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v33
-; SI-NEXT: v_alignbit_b32 v16, v5, v52, 16
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v49
-; SI-NEXT: v_alignbit_b32 v0, v0, v36, 16
-; SI-NEXT: v_alignbit_b32 v4, v6, v34, 16
-; SI-NEXT: v_alignbit_b32 v12, v14, v38, 16
-; SI-NEXT: v_alignbit_b32 v20, v22, v50, 16
-; SI-NEXT: v_alignbit_b32 v24, v5, v55, 16
-; SI-NEXT: v_alignbit_b32 v28, v30, v53, 16
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v37
+; SI-NEXT: v_or_b32_e32 v8, v5, v51
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v33
+; SI-NEXT: v_or_b32_e32 v12, v5, v50
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v38
+; SI-NEXT: v_or_b32_e32 v16, v5, v53
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v34
+; SI-NEXT: v_or_b32_e32 v20, v5, v52
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v39
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v35
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v32
+; SI-NEXT: v_or_b32_e32 v24, v5, v55
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v36
+; SI-NEXT: v_or_b32_e32 v0, v0, v49
+; SI-NEXT: v_or_b32_e32 v4, v1, v48
+; SI-NEXT: v_or_b32_e32 v28, v5, v54
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
@@ -40851,78 +40347,70 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: v_alignbit_b32 v27, v28, v24, 24
; SI-NEXT: v_alignbit_b32 v26, v28, v24, 16
; SI-NEXT: v_alignbit_b32 v25, v28, v24, 8
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v32
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v33
; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v37
; SI-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v49
; SI-NEXT: v_lshrrev_b32_e32 v29, 8, v28
+; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
+; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
+; SI-NEXT: v_bfe_u32 v23, v22, 8, 8
+; SI-NEXT: v_bfe_u32 v31, v30, 8, 8
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr49
; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr49
; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB108_2
; SI-NEXT: .LBB108_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v39
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v55
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_alignbit_b32 v24, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v49
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v36
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v54
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v31
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v51
; SI-NEXT: v_alignbit_b32 v28, v30, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v38
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v53
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_alignbit_b32 v16, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v34
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v52
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v23
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v39
; SI-NEXT: v_alignbit_b32 v20, v22, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v37
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v51
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_alignbit_b32 v8, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v33
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v50
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v15
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v35
; SI-NEXT: v_alignbit_b32 v12, v14, v0, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v35
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v49
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v32
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v32
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v48
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
; SI-NEXT: v_alignbit_b32 v4, v6, v1, 16
@@ -41930,222 +41418,211 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; SI-LABEL: bitcast_v16bf16_to_v32i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: s_and_b32 s4, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s23, 16
-; SI-NEXT: s_and_b32 s6, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s22, 16
-; SI-NEXT: s_and_b32 s8, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s21, 16
-; SI-NEXT: s_and_b32 s10, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s20, 16
-; SI-NEXT: s_and_b32 s12, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s19, 16
-; SI-NEXT: s_and_b32 s14, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s18, 16
-; SI-NEXT: s_and_b32 s18, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s19, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshr_b32 s77, s23, 16
+; SI-NEXT: s_lshr_b32 s79, s22, 16
+; SI-NEXT: s_lshr_b32 s75, s21, 16
+; SI-NEXT: s_lshr_b32 s78, s20, 16
+; SI-NEXT: s_lshr_b32 s73, s19, 16
+; SI-NEXT: s_lshr_b32 s76, s18, 16
+; SI-NEXT: s_lshr_b32 s72, s17, 16
+; SI-NEXT: s_lshr_b32 s74, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s4
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v10
-; SI-NEXT: v_lshr_b64 v[50:51], v[18:19], 16
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v28
-; SI-NEXT: v_lshr_b64 v[0:1], v[24:25], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; SI-NEXT: v_mov_b32_e32 v7, v52
-; SI-NEXT: v_lshr_b64 v[51:52], v[21:22], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_mov_b32_e32 v52, v7
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v26
-; SI-NEXT: v_lshr_b64 v[53:54], v[40:41], 16
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v52
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
-; SI-NEXT: v_lshr_b64 v[8:9], v[32:33], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; SI-NEXT: v_mov_b32_e32 v7, v55
-; SI-NEXT: v_lshr_b64 v[54:55], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v4, v2
-; SI-NEXT: v_lshr_b64 v[34:35], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v2, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[35:36], v[50:51], 16
-; SI-NEXT: v_mov_b32_e32 v55, v7
-; SI-NEXT: v_mov_b32_e32 v7, v28
-; SI-NEXT: v_lshr_b64 v[27:28], v[53:54], 24
-; SI-NEXT: v_mov_b32_e32 v10, v2
-; SI-NEXT: v_mov_b32_e32 v2, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[8:9], 24
-; SI-NEXT: v_mov_b32_e32 v28, v7
-; SI-NEXT: v_mov_b32_e32 v7, v37
-; SI-NEXT: v_lshr_b64 v[36:37], v[53:54], 16
-; SI-NEXT: v_lshr_b64 v[48:49], v[0:1], 8
-; SI-NEXT: v_mov_b32_e32 v12, v2
-; SI-NEXT: v_lshr_b64 v[16:17], v[8:9], 16
-; SI-NEXT: v_lshr_b64 v[38:39], v[8:9], 8
-; SI-NEXT: v_lshr_b64 v[19:20], v[50:51], 24
-; SI-NEXT: v_mov_b32_e32 v37, v7
-; SI-NEXT: v_mov_b32_e32 v7, v26
-; SI-NEXT: v_lshr_b64 v[25:26], v[53:54], 8
-; SI-NEXT: v_mov_b32_e32 v2, v18
-; SI-NEXT: v_lshr_b64 v[17:18], v[50:51], 8
-; SI-NEXT: v_mov_b32_e32 v26, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; SI-NEXT: v_lshrrev_b32_e32 v20, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v28
-; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v9
-; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v52
-; SI-NEXT: v_lshrrev_b32_e32 v39, 8, v51
-; SI-NEXT: v_lshrrev_b32_e32 v49, 8, v54
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_or_b32 s10, s4, s5
+; SI-NEXT: s_and_b32 s4, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_or_b32 s11, s4, s5
+; SI-NEXT: s_lshr_b64 s[4:5], s[10:11], 24
+; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 16
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s76, 16
+; SI-NEXT: s_or_b32 s26, s5, s7
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s73, 16
+; SI-NEXT: s_or_b32 s27, s5, s7
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s78, 16
+; SI-NEXT: s_or_b32 s44, s5, s7
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s75, 16
+; SI-NEXT: s_or_b32 s45, s5, s7
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s79, 16
+; SI-NEXT: s_or_b32 s60, s5, s7
+; SI-NEXT: s_and_b32 s5, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s77, 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 8
+; SI-NEXT: s_lshr_b64 s[12:13], s[26:27], 24
+; SI-NEXT: s_lshr_b64 s[14:15], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[26:27], 8
+; SI-NEXT: s_lshr_b64 s[28:29], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[40:41], s[44:45], 16
+; SI-NEXT: s_or_b32 s61, s5, s7
+; SI-NEXT: s_lshr_b64 s[42:43], s[44:45], 8
+; SI-NEXT: s_lshr_b64 s[46:47], s[60:61], 24
+; SI-NEXT: s_lshr_b64 s[56:57], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[60:61], 8
+; SI-NEXT: s_lshr_b32 s5, s11, 8
+; SI-NEXT: s_lshr_b32 s7, s27, 8
+; SI-NEXT: s_lshr_b32 s9, s45, 8
+; SI-NEXT: s_lshr_b32 s15, s61, 8
+; SI-NEXT: s_bfe_u32 s13, s72, 0x80008
+; SI-NEXT: s_bfe_u32 s25, s73, 0x80008
+; SI-NEXT: s_bfe_u32 s29, s75, 0x80008
+; SI-NEXT: s_bfe_u32 s41, s77, 0x80008
+; SI-NEXT: s_mov_b64 s[62:63], 0
; SI-NEXT: s_branch .LBB109_3
; SI-NEXT: .LBB109_2:
-; SI-NEXT: v_mov_b32_e32 v2, v18
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr41
; SI-NEXT: .LBB109_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
; SI-NEXT: s_cbranch_scc1 .LBB109_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s79, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v29, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s77, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[53:54], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v11, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s78, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v21, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s75, 16
+; SI-NEXT: v_add_f32_e32 v12, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[50:51], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v26
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v3, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s76, 16
+; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v13, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s73, 16
+; SI-NEXT: v_add_f32_e32 v15, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s74, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s5, s72, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v23
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v2
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v12
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v15
+; SI-NEXT: v_add_f32_e32 v5, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[3:4], 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v11
+; SI-NEXT: v_lshr_b64 v[17:18], v[21:22], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v18
-; SI-NEXT: v_lshr_b64 v[51:52], v[21:22], 16
; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[54:55], v[29:30], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[16:17], v[8:9], 16
-; SI-NEXT: v_lshr_b64 v[35:36], v[50:51], 16
+; SI-NEXT: v_lshr_b64 v[25:26], v[29:30], 16
+; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v11
+; SI-NEXT: v_lshr_b64 v[10:11], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[0:1], 8
+; SI-NEXT: v_lshr_b64 v[18:19], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[33:34], v[8:9], 8
+; SI-NEXT: v_lshr_b64 v[36:37], v[16:17], 8
+; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v12
; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
-; SI-NEXT: v_lshr_b64 v[48:49], v[0:1], 8
; SI-NEXT: v_lshr_b64 v[11:12], v[8:9], 24
-; SI-NEXT: v_lshr_b64 v[38:39], v[8:9], 8
-; SI-NEXT: v_lshr_b64 v[19:20], v[50:51], 24
-; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v18
-; SI-NEXT: v_lshr_b64 v[17:18], v[50:51], 8
-; SI-NEXT: v_lshr_b64 v[27:28], v[53:54], 24
-; SI-NEXT: v_lshr_b64 v[36:37], v[53:54], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[53:54], 8
-; SI-NEXT: v_lshrrev_b32_e32 v20, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v9
-; SI-NEXT: v_lshrrev_b32_e32 v39, 8, v51
-; SI-NEXT: v_lshrrev_b32_e32 v49, 8, v54
+; SI-NEXT: v_lshr_b64 v[19:20], v[16:17], 24
+; SI-NEXT: v_lshr_b64 v[48:49], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[24:25], 24
+; SI-NEXT: v_lshr_b64 v[34:35], v[24:25], 16
+; SI-NEXT: v_lshr_b64 v[37:38], v[24:25], 8
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v15
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v23
-; SI-NEXT: .LBB109_5: ; %end
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v9
+; SI-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; SI-NEXT: v_lshrrev_b32_e32 v29, 8, v25
+; SI-NEXT: s_branch .LBB109_6
+; SI-NEXT: .LBB109_5:
+; SI-NEXT: v_mov_b32_e32 v31, s41
+; SI-NEXT: v_mov_b32_e32 v23, s29
+; SI-NEXT: v_mov_b32_e32 v15, s25
+; SI-NEXT: v_mov_b32_e32 v7, s13
+; SI-NEXT: v_mov_b32_e32 v1, s11
+; SI-NEXT: v_mov_b32_e32 v0, s10
+; SI-NEXT: v_mov_b32_e32 v9, s27
+; SI-NEXT: v_mov_b32_e32 v8, s26
+; SI-NEXT: v_mov_b32_e32 v17, s45
+; SI-NEXT: v_mov_b32_e32 v16, s44
+; SI-NEXT: v_mov_b32_e32 v25, s61
+; SI-NEXT: v_mov_b32_e32 v24, s60
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v13, s7
+; SI-NEXT: v_mov_b32_e32 v21, s9
+; SI-NEXT: v_mov_b32_e32 v29, s15
+; SI-NEXT: v_mov_b32_e32 v6, s72
+; SI-NEXT: v_mov_b32_e32 v14, s73
+; SI-NEXT: v_mov_b32_e32 v22, s75
+; SI-NEXT: v_mov_b32_e32 v30, s77
+; SI-NEXT: v_mov_b32_e32 v3, s4
+; SI-NEXT: v_mov_b32_e32 v10, s6
+; SI-NEXT: v_mov_b32_e32 v26, s8
+; SI-NEXT: v_mov_b32_e32 v11, s12
+; SI-NEXT: v_mov_b32_e32 v18, s14
+; SI-NEXT: v_mov_b32_e32 v33, s24
+; SI-NEXT: v_mov_b32_e32 v19, s28
+; SI-NEXT: v_mov_b32_e32 v48, s40
+; SI-NEXT: v_mov_b32_e32 v36, s42
+; SI-NEXT: v_mov_b32_e32 v27, s46
+; SI-NEXT: v_mov_b32_e32 v34, s56
+; SI-NEXT: v_mov_b32_e32 v37, s58
+; SI-NEXT: .LBB109_6: ; %end
; SI-NEXT: v_mov_b32_e32 v4, v1
; SI-NEXT: v_mov_b32_e32 v12, v9
-; SI-NEXT: v_mov_b32_e32 v1, v48
-; SI-NEXT: v_mov_b32_e32 v2, v34
-; SI-NEXT: v_mov_b32_e32 v9, v38
-; SI-NEXT: v_mov_b32_e32 v10, v16
-; SI-NEXT: v_mov_b32_e32 v5, v20
-; SI-NEXT: v_mov_b32_e32 v13, v33
-; SI-NEXT: v_mov_b32_e32 v16, v50
-; SI-NEXT: v_mov_b32_e32 v18, v35
-; SI-NEXT: v_mov_b32_e32 v20, v51
-; SI-NEXT: v_mov_b32_e32 v21, v39
-; SI-NEXT: v_mov_b32_e32 v24, v53
-; SI-NEXT: v_mov_b32_e32 v26, v36
-; SI-NEXT: v_mov_b32_e32 v28, v54
-; SI-NEXT: v_mov_b32_e32 v29, v49
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v20, v17
+; SI-NEXT: v_mov_b32_e32 v28, v25
+; SI-NEXT: v_mov_b32_e32 v1, v26
+; SI-NEXT: v_mov_b32_e32 v2, v10
+; SI-NEXT: v_mov_b32_e32 v9, v33
+; SI-NEXT: v_mov_b32_e32 v10, v18
+; SI-NEXT: v_mov_b32_e32 v17, v36
+; SI-NEXT: v_mov_b32_e32 v18, v48
+; SI-NEXT: v_mov_b32_e32 v25, v37
+; SI-NEXT: v_mov_b32_e32 v26, v34
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16bf16_to_v32i8_scalar:
@@ -43277,275 +42754,271 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i8_to_v16bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32
-; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v3
-; SI-NEXT: v_lshlrev_b32_e32 v36, 8, v5
-; SI-NEXT: v_lshlrev_b32_e32 v32, 24, v7
-; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v11
-; SI-NEXT: v_lshlrev_b32_e32 v50, 8, v13
-; SI-NEXT: v_lshlrev_b32_e32 v48, 24, v15
-; SI-NEXT: v_lshlrev_b32_e32 v49, 24, v19
-; SI-NEXT: v_lshlrev_b32_e32 v53, 8, v21
-; SI-NEXT: v_lshlrev_b32_e32 v51, 24, v23
-; SI-NEXT: v_lshlrev_b32_e32 v52, 24, v27
-; SI-NEXT: v_lshlrev_b32_e32 v55, 8, v29
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32
+; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v5
+; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v7
+; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v35, 8, v13
+; SI-NEXT: v_lshlrev_b32_e32 v34, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v36, 24, v11
+; SI-NEXT: v_lshlrev_b32_e32 v50, 8, v21
+; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v23
+; SI-NEXT: v_lshlrev_b32_e32 v51, 24, v19
+; SI-NEXT: v_lshlrev_b32_e32 v53, 8, v29
+; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v27
+; SI-NEXT: v_lshlrev_b32_e32 v19, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v9
+; SI-NEXT: v_lshlrev_b32_e32 v49, 8, v17
+; SI-NEXT: v_lshlrev_b32_e32 v55, 8, v25
+; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v38
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v34
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: v_lshlrev_b32_e32 v52, 24, v39
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr39
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT: v_or_b32_e32 v3, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v5, v31, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v36
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v11, v32, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v14
+; SI-NEXT: v_or_b32_e32 v0, v0, v19
+; SI-NEXT: v_or_b32_e32 v2, v33, v2
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v6
+; SI-NEXT: v_or_b32_e32 v6, v34, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v10
+; SI-NEXT: v_or_b32_e32 v23, v0, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v9
-; SI-NEXT: v_or_b32_e32 v13, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v15, v35, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v12
-; SI-NEXT: v_or_b32_e32 v0, v0, v50
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v21, v48, v0
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v22
+; SI-NEXT: v_or_b32_e32 v0, v0, v27
+; SI-NEXT: v_or_b32_e32 v9, v36, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v5, v5, v50
+; SI-NEXT: v_or_b32_e32 v10, v37, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v18
+; SI-NEXT: v_or_b32_e32 v15, v0, v9
; SI-NEXT: v_and_b32_e32 v0, 0xff, v16
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v17
-; SI-NEXT: v_or_b32_e32 v23, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v18
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v27, v49, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v20
-; SI-NEXT: v_or_b32_e32 v0, v0, v53
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v22
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v33, v51, v0
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_or_b32_e32 v0, v0, v49
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v4, v31, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v12
+; SI-NEXT: v_or_b32_e32 v5, v5, v10
+; SI-NEXT: v_or_b32_e32 v11, v51, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v28
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v30
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v32
+; SI-NEXT: v_or_b32_e32 v3, v3, v35
+; SI-NEXT: v_alignbit_b32 v21, v5, v11, 16
+; SI-NEXT: v_or_b32_e32 v7, v7, v53
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v14, 0xff, v26
+; SI-NEXT: v_or_b32_e32 v11, v0, v11
; SI-NEXT: v_and_b32_e32 v0, 0xff, v24
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v25
-; SI-NEXT: v_or_b32_e32 v34, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v26
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v37, v52, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v28
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_or_b32_e32 v12, v52, v12
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v0, v0, v55
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v30
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v39, v54, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_or_b32_e32 v3, v3, v6
+; SI-NEXT: v_or_b32_e32 v7, v7, v12
+; SI-NEXT: v_or_b32_e32 v14, v54, v14
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v13, v1, v2, 16
+; SI-NEXT: v_alignbit_b32 v17, v3, v9, 16
+; SI-NEXT: v_alignbit_b32 v25, v7, v14, 16
+; SI-NEXT: v_or_b32_e32 v9, v0, v14
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v12
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr28
; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: .LBB110_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v28
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v24
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v26
+; SI-NEXT: v_or_b32_e32 v1, v55, v1
; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v30
-; SI-NEXT: v_or_b32_e32 v3, v55, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x300, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v54, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, 0x300, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v54, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: s_mov_b32 s7, 0x3000000
-; SI-NEXT: v_add_i32_e32 v28, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v24
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v25
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v26
+; SI-NEXT: v_add_i32_e32 v9, vcc, s7, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v28
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v30
; SI-NEXT: s_movk_i32 s6, 0x300
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v52, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v24, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v20
+; SI-NEXT: v_or_b32_e32 v1, v53, v1
; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v22
-; SI-NEXT: v_or_b32_e32 v3, v53, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v51, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v20, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v16
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v52, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_add_i32_e32 v7, vcc, s7, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v16
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v18
+; SI-NEXT: v_or_b32_e32 v1, v49, v1
; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v17
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v18
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v49, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v16, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v12
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v51, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_add_i32_e32 v11, vcc, s7, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v20
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v22
+; SI-NEXT: v_or_b32_e32 v1, v50, v1
; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v14
-; SI-NEXT: v_or_b32_e32 v3, v50, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v48, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v12, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v8
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v37, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_add_i32_e32 v5, vcc, s7, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v8
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v10
+; SI-NEXT: v_or_b32_e32 v1, v27, v1
; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v9
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v10
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v35, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v36, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_add_i32_e32 v15, vcc, s7, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v12
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v14
+; SI-NEXT: v_or_b32_e32 v1, v35, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v3, v34, v3
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_add_i32_e32 v8, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v4
+; SI-NEXT: v_add_i32_e32 v3, vcc, s7, v1
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v6
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v2
-; SI-NEXT: v_or_b32_e32 v3, v36, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v0, v19, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v33, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v23, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v6
+; SI-NEXT: v_or_b32_e32 v0, v32, v0
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v4, v32, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v1, v31, v1
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_add_i32_e32 v4, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v8
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v12
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v16
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v20
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v24
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v28
+; SI-NEXT: v_add_i32_e32 v1, vcc, s7, v0
+; SI-NEXT: v_alignbit_b32 v13, v1, v23, 16
+; SI-NEXT: v_alignbit_b32 v17, v3, v15, 16
+; SI-NEXT: v_alignbit_b32 v21, v5, v11, 16
+; SI-NEXT: v_alignbit_b32 v25, v7, v9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v7
; SI-NEXT: .LBB110_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v13
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v19
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v23
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v29
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v17
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v38
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v21
+; SI-NEXT: v_or_b32_e32 v4, v4, v6
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v39
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v25
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v48
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v32i8_to_v16bf16:
@@ -44277,270 +43750,278 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_readfirstlane_b32 s63, v17
-; SI-NEXT: v_readfirstlane_b32 s72, v16
-; SI-NEXT: v_readfirstlane_b32 s77, v15
-; SI-NEXT: v_readfirstlane_b32 s78, v14
-; SI-NEXT: v_readfirstlane_b32 s56, v13
-; SI-NEXT: v_readfirstlane_b32 s57, v12
-; SI-NEXT: v_readfirstlane_b32 s61, v11
-; SI-NEXT: v_readfirstlane_b32 s73, v10
-; SI-NEXT: v_readfirstlane_b32 s40, v9
-; SI-NEXT: v_readfirstlane_b32 s41, v8
-; SI-NEXT: v_readfirstlane_b32 s45, v7
-; SI-NEXT: v_readfirstlane_b32 s58, v6
-; SI-NEXT: v_readfirstlane_b32 s9, v5
-; SI-NEXT: v_readfirstlane_b32 s10, v4
-; SI-NEXT: v_readfirstlane_b32 s13, v3
-; SI-NEXT: v_readfirstlane_b32 s42, v2
-; SI-NEXT: v_readfirstlane_b32 s6, v1
+; SI-NEXT: v_readfirstlane_b32 s62, v17
+; SI-NEXT: v_readfirstlane_b32 s63, v16
+; SI-NEXT: v_readfirstlane_b32 s73, v15
+; SI-NEXT: v_readfirstlane_b32 s75, v14
+; SI-NEXT: v_readfirstlane_b32 s76, v13
+; SI-NEXT: v_readfirstlane_b32 s77, v12
+; SI-NEXT: v_readfirstlane_b32 s78, v11
+; SI-NEXT: v_readfirstlane_b32 s79, v10
+; SI-NEXT: v_readfirstlane_b32 s56, v9
+; SI-NEXT: v_readfirstlane_b32 s57, v8
+; SI-NEXT: v_readfirstlane_b32 s58, v7
+; SI-NEXT: v_readfirstlane_b32 s59, v6
+; SI-NEXT: v_readfirstlane_b32 s60, v5
+; SI-NEXT: v_readfirstlane_b32 s61, v4
+; SI-NEXT: v_readfirstlane_b32 s72, v3
+; SI-NEXT: v_readfirstlane_b32 s74, v2
+; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s7, v0
+; SI-NEXT: v_readfirstlane_b32 s47, v0
; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: s_and_b32 s8, s20, 0xff
+; SI-NEXT: s_lshl_b32 s9, s21, 8
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s22, 0xff
; SI-NEXT: s_and_b32 s4, s16, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_or_b32 s8, s5, s4
-; SI-NEXT: s_and_b32 s4, s18, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s19, 24
-; SI-NEXT: s_or_b32 s12, s5, s4
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s21, 8
+; SI-NEXT: s_lshl_b32 s5, s17, 8
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_lshl_b32 s11, s23, 24
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s11, s4, 16
-; SI-NEXT: s_and_b32 s4, s22, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s23, 24
-; SI-NEXT: s_or_b32 s15, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xff
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_or_b32 s44, s11, s9
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s19, 24
+; SI-NEXT: s_or_b32 s13, s8, s44
+; SI-NEXT: s_and_b32 s8, s28, 0xff
+; SI-NEXT: s_lshl_b32 s9, s29, 8
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s12, s6, s5
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s47, 0xff
+; SI-NEXT: s_or_b32 s10, s4, s12
; SI-NEXT: s_and_b32 s4, s24, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s25, 24
-; SI-NEXT: s_or_b32 s14, s5, s4
-; SI-NEXT: s_and_b32 s4, s26, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s27, 24
-; SI-NEXT: s_or_b32 s44, s5, s4
-; SI-NEXT: s_and_b32 s4, s28, 0xff
-; SI-NEXT: s_lshl_b32 s5, s29, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s43, s4, 16
-; SI-NEXT: s_and_b32 s4, s7, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s6, 24
-; SI-NEXT: s_or_b32 s47, s5, s4
-; SI-NEXT: s_and_b32 s4, s42, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s13, 24
-; SI-NEXT: s_or_b32 s46, s5, s4
-; SI-NEXT: s_and_b32 s4, s10, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s9, 24
-; SI-NEXT: s_or_b32 s60, s5, s4
-; SI-NEXT: s_and_b32 s4, s58, 0xff
-; SI-NEXT: s_lshl_b32 s5, s45, 8
+; SI-NEXT: s_lshl_b32 s5, s25, 8
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_lshl_b32 s11, s46, 24
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s59, s4, 16
-; SI-NEXT: s_and_b32 s4, s41, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s40, 24
-; SI-NEXT: s_or_b32 s74, s5, s4
-; SI-NEXT: s_and_b32 s4, s73, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s61, 24
-; SI-NEXT: s_or_b32 s62, s5, s4
-; SI-NEXT: s_and_b32 s4, s57, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s56, 24
-; SI-NEXT: s_or_b32 s76, s5, s4
-; SI-NEXT: s_and_b32 s4, s78, 0xff
-; SI-NEXT: s_lshl_b32 s5, s77, 8
+; SI-NEXT: s_and_b32 s5, s26, 0xff
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_or_b32 s45, s11, s9
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s27, 24
+; SI-NEXT: s_or_b32 s15, s8, s45
+; SI-NEXT: s_and_b32 s8, s59, 0xff
+; SI-NEXT: s_lshl_b32 s9, s58, 8
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s14, s6, s5
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s57, 0xff
+; SI-NEXT: s_or_b32 s6, s4, s14
+; SI-NEXT: s_and_b32 s4, s74, 0xff
+; SI-NEXT: s_lshl_b32 s5, s72, 8
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_lshl_b32 s11, s56, 24
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s75, s4, 16
-; SI-NEXT: s_and_b32 s4, s72, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s63, 24
-; SI-NEXT: s_or_b32 s79, s5, s4
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s61, 0xff
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_or_b32 s88, s11, s9
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s7, s60, 24
+; SI-NEXT: s_or_b32 s41, s8, s88
+; SI-NEXT: s_and_b32 s8, s75, 0xff
+; SI-NEXT: s_lshl_b32 s9, s73, 8
+; SI-NEXT: s_or_b32 s40, s7, s5
+; SI-NEXT: s_and_b32 s5, s79, 0xff
+; SI-NEXT: s_lshl_b32 s7, s78, 8
+; SI-NEXT: s_or_b32 s8, s8, s9
+; SI-NEXT: s_and_b32 s9, s63, 0xff
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_and_b32 s7, s77, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_lshl_b32 s11, s62, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_or_b32 s89, s11, s9
+; SI-NEXT: s_lshl_b32 s9, s76, 24
+; SI-NEXT: s_or_b32 s42, s9, s7
+; SI-NEXT: s_and_b32 s7, s8, 0xffff
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s43, s7, s89
+; SI-NEXT: s_or_b32 s4, s4, s40
+; SI-NEXT: s_or_b32 s8, s5, s42
+; SI-NEXT: s_mov_b32 s11, s13
+; SI-NEXT: s_lshr_b64 s[12:13], s[12:13], 16
+; SI-NEXT: s_mov_b32 s7, s15
+; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], 16
+; SI-NEXT: s_mov_b32 s5, s41
+; SI-NEXT: s_lshr_b64 s[40:41], s[40:41], 16
+; SI-NEXT: s_mov_b32 s9, s43
+; SI-NEXT: s_lshr_b64 s[42:43], s[42:43], 16
+; SI-NEXT: s_lshr_b32 s15, s44, 16
+; SI-NEXT: s_lshr_b32 s41, s45, 16
+; SI-NEXT: s_lshr_b32 s43, s88, 16
+; SI-NEXT: s_lshr_b32 s13, s89, 16
+; SI-NEXT: s_mov_b64 s[44:45], 0
; SI-NEXT: s_branch .LBB111_3
; SI-NEXT: .LBB111_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: s_mov_b64 s[44:45], -1
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr13
; SI-NEXT: .LBB111_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; SI-NEXT: s_cselect_b32 s44, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s44, 1
; SI-NEXT: s_cbranch_scc1 .LBB111_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s78, s78, 3
-; SI-NEXT: s_and_b32 s4, s78, 0xff
-; SI-NEXT: s_lshl_b32 s5, s77, 8
-; SI-NEXT: s_add_i32 s72, s72, 3
+; SI-NEXT: s_add_i32 s79, s79, 3
+; SI-NEXT: s_and_b32 s4, s79, 0xff
+; SI-NEXT: s_lshl_b32 s5, s78, 8
+; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s8, s72, 0xff
+; SI-NEXT: s_and_b32 s6, s77, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s63, 24
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s76, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_add_i32 s73, s73, 3
+; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s73, 0xff
-; SI-NEXT: s_lshl_b32 s8, s61, 8
+; SI-NEXT: s_add_i32 s75, s75, 3
+; SI-NEXT: s_add_i32 s8, s4, 0x3000000
+; SI-NEXT: s_and_b32 s4, s75, 0xff
+; SI-NEXT: s_lshl_b32 s5, s73, 8
+; SI-NEXT: s_add_i32 s63, s63, 3
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s6, s63, 0xff
+; SI-NEXT: s_addk_i32 s4, 0x300
+; SI-NEXT: s_lshl_b32 s5, s62, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s74, s74, 3
+; SI-NEXT: s_add_i32 s9, s4, 0x3000000
+; SI-NEXT: s_and_b32 s4, s74, 0xff
+; SI-NEXT: s_lshl_b32 s5, s72, 8
+; SI-NEXT: s_add_i32 s61, s61, 3
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s6, s61, 0xff
+; SI-NEXT: s_addk_i32 s4, 0x300
+; SI-NEXT: s_lshl_b32 s5, s60, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_add_i32 s59, s59, 3
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s59, 0xff
+; SI-NEXT: s_lshl_b32 s6, s58, 8
; SI-NEXT: s_add_i32 s57, s57, 3
-; SI-NEXT: s_or_b32 s5, s8, s5
-; SI-NEXT: s_and_b32 s11, s57, 0xff
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s7, s57, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s8, s56, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s11
-; SI-NEXT: s_add_i32 s58, s58, 3
-; SI-NEXT: s_or_b32 s5, s8, s5
-; SI-NEXT: s_and_b32 s8, s58, 0xff
-; SI-NEXT: s_lshl_b32 s11, s45, 8
-; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: s_or_b32 s8, s11, s8
-; SI-NEXT: s_and_b32 s12, s41, 0xff
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_lshl_b32 s11, s40, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_or_b32 s8, s11, s8
-; SI-NEXT: s_add_i32 s42, s42, 3
-; SI-NEXT: s_add_i32 s40, s8, 0x3000000
-; SI-NEXT: s_and_b32 s8, s42, 0xff
-; SI-NEXT: s_lshl_b32 s11, s13, 8
-; SI-NEXT: s_add_i32 s10, s10, 3
-; SI-NEXT: s_or_b32 s8, s11, s8
-; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_lshl_b32 s9, s9, 24
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_add_i32 s9, s8, 0x3000000
-; SI-NEXT: s_and_b32 s8, s28, 0xff
-; SI-NEXT: s_lshl_b32 s10, s29, 8
-; SI-NEXT: s_add_i32 s7, s7, 3
-; SI-NEXT: s_or_b32 s8, s10, s8
-; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_lshl_b32 s6, s56, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_and_b32 s7, s24, 0xff
-; SI-NEXT: s_lshl_b32 s8, s25, 8
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s24, 0xff
+; SI-NEXT: s_lshl_b32 s7, s25, 8
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s10, s26, 0xff
-; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s8, s27, 24
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s27, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s20, 0xff
-; SI-NEXT: s_lshl_b32 s10, s21, 8
-; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s8, s10, s8
-; SI-NEXT: s_and_b32 s11, s22, 0xff
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_lshl_b32 s10, s23, 24
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s10
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s28, 0xff
+; SI-NEXT: s_lshl_b32 s10, s29, 8
+; SI-NEXT: s_add_i32 s47, s47, 3
+; SI-NEXT: s_or_b32 s7, s10, s7
+; SI-NEXT: s_and_b32 s11, s47, 0xff
+; SI-NEXT: s_addk_i32 s7, 0x300
+; SI-NEXT: s_lshl_b32 s10, s46, 24
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_or_b32 s8, s10, s8
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s10, s8, 0x3000000
-; SI-NEXT: s_and_b32 s8, s16, 0xff
+; SI-NEXT: s_or_b32 s7, s10, s7
+; SI-NEXT: s_and_b32 s10, s16, 0xff
; SI-NEXT: s_lshl_b32 s11, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s8, s11, s8
+; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s12, s18, 0xff
-; SI-NEXT: s_addk_i32 s8, 0x300
+; SI-NEXT: s_addk_i32 s10, 0x300
; SI-NEXT: s_lshl_b32 s11, s19, 24
; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_or_b32 s8, s11, s8
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s11, s20, 0xff
+; SI-NEXT: s_lshl_b32 s12, s21, 8
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s13, s22, 0xff
+; SI-NEXT: s_addk_i32 s11, 0x300
+; SI-NEXT: s_lshl_b32 s12, s23, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_or_b32 s12, s12, s13
+; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: s_add_i32 s8, s8, 0x3000000
-; SI-NEXT: s_and_b32 s12, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_and_b32 s15, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s10, 16
-; SI-NEXT: s_and_b32 s44, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s7, 16
-; SI-NEXT: s_and_b32 s47, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s6, 16
-; SI-NEXT: s_and_b32 s60, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s9, 16
-; SI-NEXT: s_and_b32 s74, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s40, 16
-; SI-NEXT: s_and_b32 s76, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s5, 16
-; SI-NEXT: s_and_b32 s79, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s75, s4, 16
+; SI-NEXT: s_add_i32 s10, s10, 0x3000000
+; SI-NEXT: s_add_i32 s11, s11, 0x3000000
+; SI-NEXT: s_lshr_b64 s[12:13], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
+; SI-NEXT: s_lshr_b32 s15, s11, 16
+; SI-NEXT: s_lshr_b32 s41, s7, 16
+; SI-NEXT: s_lshr_b32 s43, s5, 16
+; SI-NEXT: s_lshr_b32 s13, s9, 16
; SI-NEXT: .LBB111_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s11
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s44
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s47
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s43
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s60
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s46
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s74
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s59
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s76
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s79
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s75
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s10, s10, s12
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s15, 16
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s14, 16
+; SI-NEXT: s_or_b32 s6, s6, s12
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s41, 16
+; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s40, 16
+; SI-NEXT: s_or_b32 s4, s4, s12
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s43, 16
+; SI-NEXT: s_or_b32 s5, s5, s12
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s42, 16
+; SI-NEXT: s_or_b32 s8, s8, s12
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s13, 16
+; SI-NEXT: s_or_b32 s9, s9, s12
+; SI-NEXT: v_mov_b32_e32 v0, s10
+; SI-NEXT: v_mov_b32_e32 v1, s11
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: v_mov_b32_e32 v5, s5
+; SI-NEXT: v_mov_b32_e32 v6, s8
+; SI-NEXT: v_mov_b32_e32 v7, s9
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v32i8_to_v16bf16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 48259e19a00c0..136c1e0744923 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -1148,25 +1148,20 @@ define <2 x bfloat> @bitcast_i32_to_v2bf16(i32 %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_i32_to_v2bf16:
@@ -1232,14 +1227,12 @@ define inreg <2 x bfloat> @bitcast_i32_to_v2bf16_scalar(i32 inreg %a, i32 inreg
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB13_3
; SI-NEXT: .LBB13_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB13_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -1247,13 +1240,12 @@ define inreg <2 x bfloat> @bitcast_i32_to_v2bf16_scalar(i32 inreg %a, i32 inreg
; SI-NEXT: s_cbranch_scc1 .LBB13_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: .LBB13_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_i32_to_v2bf16_scalar:
@@ -1339,11 +1331,10 @@ define i32 @bitcast_v2bf16_to_i32(<2 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2bf16_to_i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v2, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -1355,15 +1346,14 @@ define i32 @bitcast_v2bf16_to_i32(<2 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB14_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB14_2
; SI-NEXT: .LBB14_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
@@ -1523,33 +1513,34 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; SI-LABEL: bitcast_v2bf16_to_i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB15_3
; SI-NEXT: .LBB15_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB15_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB15_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v1, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v1
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: .LBB15_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB15_5:
+; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2bf16_to_i32_scalar:
@@ -3611,25 +3602,20 @@ define <2 x bfloat> @bitcast_f32_to_v2bf16(float %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_f32_to_v2bf16:
@@ -3695,13 +3681,11 @@ define inreg <2 x bfloat> @bitcast_f32_to_v2bf16_scalar(float inreg %a, i32 inre
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB33_3
; SI-NEXT: .LBB33_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: .LBB33_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
@@ -3710,17 +3694,15 @@ define inreg <2 x bfloat> @bitcast_f32_to_v2bf16_scalar(float inreg %a, i32 inre
; SI-NEXT: s_cbranch_scc1 .LBB33_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: s_branch .LBB33_6
; SI-NEXT: .LBB33_5:
-; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: .LBB33_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_f32_to_v2bf16_scalar:
@@ -3808,11 +3790,10 @@ define float @bitcast_v2bf16_to_f32(<2 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2bf16_to_f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v2, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -3824,15 +3805,14 @@ define float @bitcast_v2bf16_to_f32(<2 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB34_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB34_2
; SI-NEXT: .LBB34_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
@@ -3992,33 +3972,34 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; SI-LABEL: bitcast_v2bf16_to_f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB35_3
; SI-NEXT: .LBB35_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB35_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB35_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v1, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v1
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: .LBB35_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB35_5:
+; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2bf16_to_f32_scalar:
@@ -5702,20 +5683,28 @@ define <2 x bfloat> @bitcast_v2i16_to_v2bf16(<2 x i16> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x30000, v0
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x30000, v1
-; SI-NEXT: ; %bb.2: ; %end
+; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2i16_to_v2bf16:
@@ -5781,33 +5770,35 @@ define inreg <2 x bfloat> @bitcast_v2i16_to_v2bf16_scalar(<2 x i16> inreg %a, i3
; SI-LABEL: bitcast_v2i16_to_v2bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_lshl_b32 s8, s7, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB49_3
; SI-NEXT: .LBB49_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB49_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB49_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_lshl_b32 s4, s7, 16
-; SI-NEXT: s_lshl_b32 s5, s16, 16
-; SI-NEXT: s_add_i32 s6, s5, 0x30000
-; SI-NEXT: s_add_i32 s8, s4, 0x30000
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s5, s6, 3
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s5, 16
+; SI-NEXT: s_or_b32 s7, s4, s6
+; SI-NEXT: s_and_b32 s6, s5, 0xffff
; SI-NEXT: .LBB49_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2i16_to_v2bf16_scalar:
@@ -5898,33 +5889,22 @@ define <2 x i16> @bitcast_v2bf16_to_v2i16(<2 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2bf16_to_v2i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: ; %bb.4: ; %end
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; SI-NEXT: ; %bb.2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -6073,34 +6053,32 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v2bf16_to_v2i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB51_3
; SI-NEXT: .LBB51_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: .LBB51_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB51_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v1, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v1
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
-; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: s_branch .LBB51_6
+; SI-NEXT: .LBB51_5:
+; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: v_mov_b32_e32 v2, s16
+; SI-NEXT: .LBB51_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -7441,35 +7419,33 @@ define <2 x bfloat> @bitcast_v2f16_to_v2bf16(<2 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2f16_to_v2bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB60_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v1
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
; SI-NEXT: .LBB60_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2f16_to_v2bf16:
@@ -7540,14 +7516,14 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s7, s16, 16
-; SI-NEXT: s_lshl_b32 s8, s6, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB61_3
; SI-NEXT: .LBB61_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: .LBB61_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -7557,20 +7533,19 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_branch .LBB61_6
; SI-NEXT: .LBB61_5:
-; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: .LBB61_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2f16_to_v2bf16_scalar:
@@ -7664,34 +7639,30 @@ define <2 x half> @bitcast_v2bf16_to_v2f16(<2 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2bf16_to_v2f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; SI-NEXT: v_alignbit_b32 v1, v0, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v0, v3
+; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-NEXT: v_alignbit_b32 v1, v2, v0, 16
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2bf16_to_v2f16:
@@ -7846,33 +7817,36 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; SI-LABEL: bitcast_v2bf16_to_v2f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB63_3
; SI-NEXT: .LBB63_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB63_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB63_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v1, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v1
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
-; SI-NEXT: .LBB63_5: ; %end
+; SI-NEXT: s_branch .LBB63_6
+; SI-NEXT: .LBB63_5:
+; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: v_mov_b32_e32 v2, s7
+; SI-NEXT: .LBB63_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -9228,11 +9202,10 @@ define <1 x i32> @bitcast_v2bf16_to_v1i32(<2 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2bf16_to_v1i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v2, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -9244,15 +9217,14 @@ define <1 x i32> @bitcast_v2bf16_to_v1i32(<2 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB72_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB72_2
; SI-NEXT: .LBB72_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
@@ -9412,33 +9384,34 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v2bf16_to_v1i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB73_3
; SI-NEXT: .LBB73_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB73_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB73_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v1, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v1
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: .LBB73_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB73_5:
+; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2bf16_to_v1i32_scalar:
@@ -9632,25 +9605,20 @@ define <2 x bfloat> @bitcast_v1i32_to_v2bf16(<1 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v1i32_to_v2bf16:
@@ -9716,14 +9684,12 @@ define inreg <2 x bfloat> @bitcast_v1i32_to_v2bf16_scalar(<1 x i32> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB75_3
; SI-NEXT: .LBB75_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB75_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -9731,13 +9697,12 @@ define inreg <2 x bfloat> @bitcast_v1i32_to_v2bf16_scalar(<1 x i32> inreg %a, i3
; SI-NEXT: s_cbranch_scc1 .LBB75_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: .LBB75_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v1i32_to_v2bf16_scalar:
@@ -9823,14 +9788,12 @@ define <4 x i8> @bitcast_v2bf16_to_v4i8(<2 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v2bf16_to_v4i8:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -9842,18 +9805,17 @@ define <4 x i8> @bitcast_v2bf16_to_v4i8(<2 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB76_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v2, v5, 16
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: v_bfe_u32 v3, v2, 8, 8
; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB76_2
; SI-NEXT: .LBB76_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
@@ -10074,41 +10036,43 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; SI-LABEL: bitcast_v2bf16_to_v4i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshr_b64 v[4:5], v[1:2], 16
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s7, s4, s5
+; SI-NEXT: s_lshr_b32 s8, s7, 8
+; SI-NEXT: s_bfe_u32 s9, s6, 0x80008
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB77_3
; SI-NEXT: .LBB77_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: .LBB77_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB77_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshr_b64 v[4:5], v[1:2], 16
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; SI-NEXT: .LBB77_5: ; %end
-; SI-NEXT: v_mov_b32_e32 v0, v4
-; SI-NEXT: v_mov_b32_e32 v1, v5
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v0, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_lshr_b64 v[0:1], v[1:2], 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB77_5:
+; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: v_mov_b32_e32 v2, s6
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2bf16_to_v4i8_scalar:
@@ -10351,43 +10315,46 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v1
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v1, v4, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v5, v3, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_or_b32_e32 v2, v1, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB78_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v2
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v2, v3, v1
-; SI-NEXT: v_or_b32_e32 v0, v4, v0
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x3000000, v0
-; SI-NEXT: v_add_i32_e32 v5, vcc, 0x3000000, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_or_b32_e32 v0, v5, v0
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 0x300, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 0x300, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v2
; SI-NEXT: .LBB78_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i8_to_v2bf16:
@@ -10583,13 +10550,15 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_or_b32 s6, s5, s4
-; SI-NEXT: s_and_b32 s4, s18, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s19, 24
-; SI-NEXT: s_or_b32 s7, s5, s4
+; SI-NEXT: s_lshl_b32 s5, s17, 8
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s18, 0xff
+; SI-NEXT: s_lshl_b32 s6, s19, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s5, 16
+; SI-NEXT: s_or_b32 s6, s4, s6
+; SI-NEXT: s_and_b32 s7, s5, 0xffff
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB79_3
; SI-NEXT: .LBB79_2:
@@ -10602,23 +10571,25 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB79_5
; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s4, s16, 0xff
+; SI-NEXT: s_lshl_b32 s5, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s18, 0xff
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_lshl_b32 s4, s19, 24
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xff
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s6, s5, 0x3000000
-; SI-NEXT: s_add_i32 s7, s4, 0x3000000
+; SI-NEXT: s_lshl_b32 s6, s19, 8
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_addk_i32 s4, 0x300
+; SI-NEXT: s_addk_i32 s5, 0x300
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s5, 16
+; SI-NEXT: s_or_b32 s6, s4, s6
+; SI-NEXT: s_and_b32 s7, s5, 0xffff
; SI-NEXT: .LBB79_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: s_and_b32 s4, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i8_to_v2bf16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index 8f8cd9205e24b..989464454e025 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -10,46 +10,41 @@ define <3 x half> @bitcast_v3bf16_to_v3f16(<3 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v3bf16_to_v3f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v3, v1
+; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v5, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v4, v3, v5, 16
+; SI-NEXT: v_or_b32_e32 v2, v0, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB0_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; SI-NEXT: v_alignbit_b32 v2, v2, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v2, v1, v2, 16
+; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
; SI-NEXT: .LBB0_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -245,47 +240,49 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; SI-LABEL: bitcast_v3bf16_to_v3f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshl_b32 s4, s17, 16
-; SI-NEXT: s_and_b32 s5, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v9
-; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_mov_b32 s7, s17
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s10, s17, 0xffff
+; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB1_3
; SI-NEXT: .LBB1_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB1_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB1_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: v_add_f32_e32 v0, s5, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: s_lshl_b32 s4, s17, 16
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
-; SI-NEXT: .LBB1_5: ; %end
+; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 16
+; SI-NEXT: s_branch .LBB1_6
+; SI-NEXT: .LBB1_5:
+; SI-NEXT: v_mov_b32_e32 v1, s10
+; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: .LBB1_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -527,44 +524,44 @@ define <3 x bfloat> @bitcast_v3f16_to_v3bf16(<3 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v3f16_to_v3bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_alignbit_b32 v5, v1, v0, 16
+; SI-NEXT: v_mov_b32_e32 v3, v1
+; SI-NEXT: v_alignbit_b32 v5, v3, v0, 16
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v2, v3, v1, 16
+; SI-NEXT: v_or_b32_e32 v4, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB2_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v3
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v2
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v4, v0, v2
+; SI-NEXT: v_alignbit_b32 v2, v1, v2, 16
; SI-NEXT: .LBB2_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3f16_to_v3bf16:
@@ -639,46 +636,46 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s5, s16, 16
-; SI-NEXT: s_lshl_b32 s8, s4, 16
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_mov_b32 s7, s17
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s10, s17, 0xffff
+; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB3_3
; SI-NEXT: .LBB3_2:
-; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr5
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB3_3: ; %Flow
-; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB3_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s17
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_or_b32_e32 v0, v4, v0
; SI-NEXT: s_branch .LBB3_6
; SI-NEXT: .LBB3_5:
-; SI-NEXT: v_mov_b32_e32 v2, s9
-; SI-NEXT: v_mov_b32_e32 v1, s8
+; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: .LBB3_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3f16_to_v3bf16_scalar:
@@ -778,44 +775,28 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v3bf16_to_v3i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_alignbit_b32 v3, v1, v0, 16
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB4_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
+; SI-NEXT: s_cbranch_execz .LBB4_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v2, v1, v2, 16
-; SI-NEXT: .LBB4_4: ; %end
+; SI-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; SI-NEXT: v_alignbit_b32 v3, v1, v2, 16
+; SI-NEXT: .LBB4_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -1003,43 +984,38 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v3bf16_to_v3i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshl_b32 s4, s17, 16
-; SI-NEXT: s_and_b32 s5, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB5_3
; SI-NEXT: .LBB5_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: s_mov_b64 s[6:7], -1
; SI-NEXT: .LBB5_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB5_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshr_b64 v[2:3], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s5, s16, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: v_add_f32_e32 v0, s5, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 16
-; SI-NEXT: .LBB5_5: ; %end
+; SI-NEXT: s_branch .LBB5_6
+; SI-NEXT: .LBB5_5:
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s16
+; SI-NEXT: v_mov_b32_e32 v3, s4
+; SI-NEXT: .LBB5_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -1274,34 +1250,38 @@ define <3 x bfloat> @bitcast_v3i16_to_v3bf16(<3 x i16> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v3i16_to_v3bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_alignbit_b32 v3, v1, v0, 16
+; SI-NEXT: v_mov_b32_e32 v3, v1
+; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v4, v3, v5, 16
+; SI-NEXT: v_or_b32_e32 v2, v0, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v0, v2, v0
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x30000, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x30000, v1
+; SI-NEXT: v_or_b32_e32 v0, v5, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 0x30000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v4, v0, v2, 16
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v2, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3i16_to_v3bf16:
@@ -1370,42 +1350,43 @@ define inreg <3 x bfloat> @bitcast_v3i16_to_v3bf16_scalar(<3 x i16> inreg %a, i3
; SI-LABEL: bitcast_v3i16_to_v3bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s5, s16, 16
-; SI-NEXT: s_lshl_b32 s8, s4, 16
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s6, 16
+; SI-NEXT: s_mov_b32 s9, s17
+; SI-NEXT: s_or_b32 s4, s4, s8
+; SI-NEXT: s_and_b32 s7, s17, 0xffff
+; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
+; SI-NEXT: s_mov_b64 s[10:11], 0
; SI-NEXT: s_branch .LBB7_3
; SI-NEXT: .LBB7_2:
-; SI-NEXT: s_mov_b64 s[6:7], -1
-; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB7_3: ; %Flow
-; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB7_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s5, s16, 3
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_add_i32 s4, s16, 3
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s4, s4, 0x30000
-; SI-NEXT: s_lshl_b32 s6, s17, 16
-; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s4, 16
-; SI-NEXT: s_add_i32 s9, s6, 0x30000
+; SI-NEXT: s_add_i32 s5, s17, 3
+; SI-NEXT: s_and_b32 s7, s5, 0xffff
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
; SI-NEXT: .LBB7_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3i16_to_v3bf16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 59725b5b5dea6..f4ecd35ba1592 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -4982,208 +4982,128 @@ define <32 x bfloat> @bitcast_v16i32_to_v32bf16(<16 x i32> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr28
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB20_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v22, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v24, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v27, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB20_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB20_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
+; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v22, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v24, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v27, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB20_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v26
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v24
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v27
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; SI-NEXT: v_or_b32_e32 v2, v2, v24
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v30
+; SI-NEXT: v_or_b32_e32 v4, v4, v22
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v29
+; SI-NEXT: v_or_b32_e32 v6, v6, v20
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v28
+; SI-NEXT: v_or_b32_e32 v8, v8, v19
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v26
+; SI-NEXT: v_or_b32_e32 v10, v10, v18
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v25
+; SI-NEXT: v_or_b32_e32 v12, v12, v17
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; SI-NEXT: v_or_b32_e32 v1, v1, v27
+; SI-NEXT: v_or_b32_e32 v3, v3, v24
+; SI-NEXT: v_or_b32_e32 v5, v5, v22
+; SI-NEXT: v_or_b32_e32 v7, v7, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i32_to_v32bf16:
@@ -5295,198 +5215,150 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v2
-; SI-NEXT: v_readfirstlane_b32 s78, v1
+; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s79, v0
+; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s78, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s78, 16
-; SI-NEXT: s_and_b32 s9, s79, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s79, 16
-; SI-NEXT: s_and_b32 s11, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s29, 16
-; SI-NEXT: s_and_b32 s13, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s28, 16
-; SI-NEXT: s_and_b32 s15, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s27, 16
-; SI-NEXT: s_and_b32 s41, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s26, 16
-; SI-NEXT: s_and_b32 s43, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s25, 16
-; SI-NEXT: s_and_b32 s45, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s24, 16
-; SI-NEXT: s_and_b32 s47, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s23, 16
-; SI-NEXT: s_and_b32 s57, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s22, 16
-; SI-NEXT: s_and_b32 s59, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s21, 16
-; SI-NEXT: s_and_b32 s61, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s20, 16
-; SI-NEXT: s_and_b32 s63, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s19, 16
-; SI-NEXT: s_and_b32 s73, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s18, 16
-; SI-NEXT: s_and_b32 s75, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s17, 16
-; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s56, s5, 16
+; SI-NEXT: s_lshr_b32 s57, s29, 16
+; SI-NEXT: s_lshr_b32 s58, s27, 16
+; SI-NEXT: s_lshr_b32 s59, s25, 16
+; SI-NEXT: s_lshr_b32 s60, s23, 16
+; SI-NEXT: s_lshr_b32 s61, s21, 16
+; SI-NEXT: s_lshr_b32 s62, s19, 16
+; SI-NEXT: s_lshr_b32 s63, s17, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[46:47], 0
; SI-NEXT: s_branch .LBB21_3
; SI-NEXT: .LBB21_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: s_mov_b64 s[46:47], -1
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr63
; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr61
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: .LBB21_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB21_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_add_i32 s79, s79, 3
-; SI-NEXT: s_add_i32 s78, s78, 3
-; SI-NEXT: s_and_b32 s7, s78, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s78, 16
-; SI-NEXT: s_and_b32 s9, s79, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s79, 16
-; SI-NEXT: s_and_b32 s11, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s29, 16
-; SI-NEXT: s_and_b32 s13, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s28, 16
-; SI-NEXT: s_and_b32 s15, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s27, 16
-; SI-NEXT: s_and_b32 s41, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s26, 16
-; SI-NEXT: s_and_b32 s43, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s25, 16
-; SI-NEXT: s_and_b32 s45, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s24, 16
-; SI-NEXT: s_and_b32 s47, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s23, 16
-; SI-NEXT: s_and_b32 s57, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s22, 16
-; SI-NEXT: s_and_b32 s59, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s21, 16
-; SI-NEXT: s_and_b32 s61, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s20, 16
-; SI-NEXT: s_and_b32 s63, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s19, 16
-; SI-NEXT: s_and_b32 s73, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s18, 16
-; SI-NEXT: s_and_b32 s75, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s17, 16
-; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s16, 16
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s5, s5, 3
+; SI-NEXT: s_add_i32 s4, s4, 3
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
+; SI-NEXT: s_lshr_b32 s56, s5, 16
+; SI-NEXT: s_lshr_b32 s57, s29, 16
+; SI-NEXT: s_lshr_b32 s58, s27, 16
+; SI-NEXT: s_lshr_b32 s59, s25, 16
+; SI-NEXT: s_lshr_b32 s60, s23, 16
+; SI-NEXT: s_lshr_b32 s61, s21, 16
+; SI-NEXT: s_lshr_b32 s62, s19, 16
+; SI-NEXT: s_lshr_b32 s63, s17, 16
; SI-NEXT: .LBB21_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s74
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s73
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s72
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s63
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s61
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s60
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s59
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s58
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s57
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s56
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s47
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s46
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s45
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s44
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s43
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s42
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s41
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s40
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: s_and_b32 s7, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s44, 16
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_and_b32 s9, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s63, 16
+; SI-NEXT: s_or_b32 s9, s9, s11
+; SI-NEXT: s_and_b32 s11, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s42, 16
+; SI-NEXT: s_or_b32 s11, s11, s13
+; SI-NEXT: s_and_b32 s13, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s62, 16
+; SI-NEXT: s_or_b32 s13, s13, s15
+; SI-NEXT: s_and_b32 s15, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s40, 16
+; SI-NEXT: s_or_b32 s15, s15, s16
+; SI-NEXT: s_and_b32 s16, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s61, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_and_b32 s17, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_or_b32 s14, s17, s14
+; SI-NEXT: s_and_b32 s17, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s60, 16
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_and_b32 s18, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s12, s18, s12
+; SI-NEXT: s_and_b32 s18, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s59, 16
+; SI-NEXT: s_or_b32 s18, s18, s19
+; SI-NEXT: s_and_b32 s19, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s10, s19, s10
+; SI-NEXT: s_and_b32 s19, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s58, 16
+; SI-NEXT: s_or_b32 s19, s19, s20
+; SI-NEXT: s_and_b32 s20, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s8, s20, s8
+; SI-NEXT: s_and_b32 s20, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s57, 16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s56, 16
+; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: v_mov_b32_e32 v2, s11
+; SI-NEXT: v_mov_b32_e32 v3, s13
+; SI-NEXT: v_mov_b32_e32 v4, s15
+; SI-NEXT: v_mov_b32_e32 v5, s16
+; SI-NEXT: v_mov_b32_e32 v6, s14
+; SI-NEXT: v_mov_b32_e32 v7, s17
+; SI-NEXT: v_mov_b32_e32 v8, s12
+; SI-NEXT: v_mov_b32_e32 v9, s18
+; SI-NEXT: v_mov_b32_e32 v10, s10
+; SI-NEXT: v_mov_b32_e32 v11, s19
+; SI-NEXT: v_mov_b32_e32 v12, s8
+; SI-NEXT: v_mov_b32_e32 v13, s20
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i32_to_v32bf16_scalar:
@@ -5660,71 +5532,55 @@ define <16 x i32> @bitcast_v32bf16_to_v16i32(<32 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32bf16_to_v16i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v32, v15
+; SI-NEXT: v_mov_b32_e32 v17, v14
+; SI-NEXT: v_mov_b32_e32 v18, v13
+; SI-NEXT: v_mov_b32_e32 v19, v12
+; SI-NEXT: v_mov_b32_e32 v20, v11
+; SI-NEXT: v_mov_b32_e32 v21, v10
+; SI-NEXT: v_mov_b32_e32 v22, v9
+; SI-NEXT: v_mov_b32_e32 v23, v8
+; SI-NEXT: v_mov_b32_e32 v24, v7
+; SI-NEXT: v_mov_b32_e32 v25, v6
+; SI-NEXT: v_mov_b32_e32 v26, v5
+; SI-NEXT: v_mov_b32_e32 v27, v4
+; SI-NEXT: v_mov_b32_e32 v28, v3
+; SI-NEXT: v_mov_b32_e32 v29, v2
+; SI-NEXT: v_mov_b32_e32 v30, v1
+; SI-NEXT: v_mov_b32_e32 v31, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v31
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v15
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -5736,165 +5592,149 @@ define <16 x i32> @bitcast_v32bf16_to_v16i32(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB22_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v52
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v16
-; SI-NEXT: v_alignbit_b32 v0, v0, v55, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v53, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v51, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v49, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v39, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v37, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v35, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v33, 16
-; SI-NEXT: v_alignbit_b32 v8, v8, v31, 16
-; SI-NEXT: v_alignbit_b32 v9, v9, v29, 16
-; SI-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; SI-NEXT: v_alignbit_b32 v11, v11, v25, 16
-; SI-NEXT: v_alignbit_b32 v12, v12, v23, 16
-; SI-NEXT: v_alignbit_b32 v13, v13, v21, 16
-; SI-NEXT: v_alignbit_b32 v14, v14, v19, 16
-; SI-NEXT: v_alignbit_b32 v15, v15, v17, 16
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v31
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v30
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v29
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v28
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v27
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v26
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v24
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v22
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v20
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v18
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v32
+; SI-NEXT: v_or_b32_e32 v0, v0, v55
+; SI-NEXT: v_or_b32_e32 v1, v1, v54
+; SI-NEXT: v_or_b32_e32 v2, v2, v53
+; SI-NEXT: v_or_b32_e32 v3, v3, v52
+; SI-NEXT: v_or_b32_e32 v4, v4, v51
+; SI-NEXT: v_or_b32_e32 v5, v5, v50
+; SI-NEXT: v_or_b32_e32 v6, v6, v49
+; SI-NEXT: v_or_b32_e32 v7, v7, v48
+; SI-NEXT: v_or_b32_e32 v8, v8, v39
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v37
+; SI-NEXT: v_or_b32_e32 v11, v11, v36
+; SI-NEXT: v_or_b32_e32 v12, v12, v35
+; SI-NEXT: v_or_b32_e32 v13, v13, v34
+; SI-NEXT: v_or_b32_e32 v14, v14, v33
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr52
; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB22_2
; SI-NEXT: .LBB22_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v52
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v31
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v55
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v54
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v53
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v28
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v52
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v27
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v36
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v26
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v50
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v34
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v25
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v49
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v32
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v48
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v30
; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v23
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v39
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v26
; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v21
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v37
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v24
; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v20
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v36
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v22
; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v35
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v20
; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v18
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v34
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v18
; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v33
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v32
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
@@ -7075,251 +6915,226 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-LABEL: bitcast_v32bf16_to_v16i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v2
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s14
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
+; SI-NEXT: v_writelane_b32 v18, s36, 0
+; SI-NEXT: v_writelane_b32 v18, s37, 1
+; SI-NEXT: v_writelane_b32 v18, s38, 2
+; SI-NEXT: v_writelane_b32 v18, s39, 3
+; SI-NEXT: v_writelane_b32 v18, s48, 4
+; SI-NEXT: v_writelane_b32 v18, s49, 5
+; SI-NEXT: v_writelane_b32 v18, s50, 6
+; SI-NEXT: v_writelane_b32 v18, s51, 7
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s8, s29, 16
+; SI-NEXT: s_lshr_b32 s11, s28, 16
+; SI-NEXT: s_lshr_b32 s12, s27, 16
+; SI-NEXT: s_lshr_b32 s13, s26, 16
+; SI-NEXT: s_lshr_b32 s14, s25, 16
+; SI-NEXT: s_lshr_b32 s15, s24, 16
+; SI-NEXT: s_lshr_b32 s56, s23, 16
+; SI-NEXT: s_lshr_b32 s57, s22, 16
+; SI-NEXT: s_lshr_b32 s58, s21, 16
+; SI-NEXT: s_lshr_b32 s59, s20, 16
+; SI-NEXT: s_lshr_b32 s60, s19, 16
+; SI-NEXT: s_lshr_b32 s61, s18, 16
+; SI-NEXT: s_lshr_b32 s62, s17, 16
+; SI-NEXT: s_lshr_b32 s63, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s7, 16
+; SI-NEXT: s_lshr_b32 s9, s10, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v34
-; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v32
-; SI-NEXT: v_lshr_b64 v[2:3], v[38:39], 16
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v30
-; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
-; SI-NEXT: v_mov_b32_e32 v7, v38
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v26
-; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
-; SI-NEXT: v_mov_b32_e32 v8, v36
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v24
-; SI-NEXT: v_mov_b32_e32 v38, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[35:36], 16
-; SI-NEXT: v_mov_b32_e32 v9, v34
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v22
-; SI-NEXT: v_mov_b32_e32 v36, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v10, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v20
-; SI-NEXT: v_mov_b32_e32 v34, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[31:32], 16
-; SI-NEXT: v_mov_b32_e32 v11, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v44
-; SI-NEXT: v_mov_b32_e32 v32, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v12, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v43
-; SI-NEXT: v_mov_b32_e32 v30, v11
-; SI-NEXT: v_lshr_b64 v[10:11], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v13, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v18
-; SI-NEXT: v_mov_b32_e32 v28, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v14, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v42
-; SI-NEXT: v_mov_b32_e32 v26, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v15, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v41
-; SI-NEXT: v_mov_b32_e32 v24, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[21:22], 16
-; SI-NEXT: v_mov_b32_e32 v39, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v40
-; SI-NEXT: v_mov_b32_e32 v22, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v20, v39
-; SI-NEXT: v_mov_b32_e32 v39, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
-; SI-NEXT: v_mov_b32_e32 v49, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v18, v39
-; SI-NEXT: v_mov_b32_e32 v16, v49
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s63, 16
+; SI-NEXT: s_or_b32 s36, s4, s5
+; SI-NEXT: s_and_b32 s4, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_or_b32 s37, s4, s5
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_or_b32 s38, s4, s5
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_or_b32 s39, s4, s5
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_or_b32 s40, s4, s5
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_or_b32 s41, s4, s5
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_or_b32 s42, s4, s5
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_or_b32 s43, s4, s5
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_or_b32 s44, s4, s5
+; SI-NEXT: s_and_b32 s4, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s45, s4, s5
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s46, s4, s5
+; SI-NEXT: s_and_b32 s4, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s47, s4, s5
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_or_b32 s48, s4, s5
+; SI-NEXT: s_and_b32 s4, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_or_b32 s49, s4, s5
+; SI-NEXT: s_and_b32 s4, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s50, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB23_3
; SI-NEXT: .LBB23_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; SI-NEXT: .LBB23_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB23_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v16, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s63, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s62, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s4, v16
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v16
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v16
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_add_f32_e32 v7, s4, v16
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; SI-NEXT: v_add_f32_e32 v8, s4, v16
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v44
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
+; SI-NEXT: v_add_f32_e32 v9, s4, v16
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v43
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, s4, v16
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
+; SI-NEXT: v_add_f32_e32 v11, s4, v16
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v42
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, s4, v16
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v41
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, s4, v16
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v40
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v14, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, s4, v16
+; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v17, s5, v16
+; SI-NEXT: v_add_f32_e32 v15, s4, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v17
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB23_5: ; %end
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: s_branch .LBB23_6
+; SI-NEXT: .LBB23_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: .LBB23_6: ; %end
+; SI-NEXT: v_readlane_b32 s51, v18, 7
+; SI-NEXT: v_readlane_b32 s50, v18, 6
+; SI-NEXT: v_readlane_b32 s49, v18, 5
+; SI-NEXT: v_readlane_b32 s48, v18, 4
+; SI-NEXT: v_readlane_b32 s39, v18, 3
+; SI-NEXT: v_readlane_b32 s38, v18, 2
+; SI-NEXT: v_readlane_b32 s37, v18, 1
+; SI-NEXT: v_readlane_b32 s36, v18, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -19883,208 +19698,128 @@ define <32 x bfloat> @bitcast_v16f32_to_v32bf16(<16 x float> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr28
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB44_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v22, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v24, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v27, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB44_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB44_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
+; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_add_f32_e32 v4, 1.0, v4
+; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
; SI-NEXT: v_add_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_add_f32_e32 v6, 1.0, v6
+; SI-NEXT: v_add_f32_e32 v4, 1.0, v4
; SI-NEXT: v_add_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_add_f32_e32 v8, 1.0, v8
+; SI-NEXT: v_add_f32_e32 v6, 1.0, v6
; SI-NEXT: v_add_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_add_f32_e32 v10, 1.0, v10
+; SI-NEXT: v_add_f32_e32 v8, 1.0, v8
; SI-NEXT: v_add_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_add_f32_e32 v12, 1.0, v12
+; SI-NEXT: v_add_f32_e32 v10, 1.0, v10
; SI-NEXT: v_add_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_add_f32_e32 v14, 1.0, v14
+; SI-NEXT: v_add_f32_e32 v12, 1.0, v12
; SI-NEXT: v_add_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
+; SI-NEXT: v_add_f32_e32 v14, 1.0, v14
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v22, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v24, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v27, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB44_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v26
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v24
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v27
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; SI-NEXT: v_or_b32_e32 v2, v2, v24
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v30
+; SI-NEXT: v_or_b32_e32 v4, v4, v22
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v29
+; SI-NEXT: v_or_b32_e32 v6, v6, v20
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v28
+; SI-NEXT: v_or_b32_e32 v8, v8, v19
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v26
+; SI-NEXT: v_or_b32_e32 v10, v10, v18
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v25
+; SI-NEXT: v_or_b32_e32 v12, v12, v17
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; SI-NEXT: v_or_b32_e32 v1, v1, v27
+; SI-NEXT: v_or_b32_e32 v3, v3, v24
+; SI-NEXT: v_or_b32_e32 v5, v5, v22
+; SI-NEXT: v_or_b32_e32 v7, v7, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f32_to_v32bf16:
@@ -20188,232 +19923,168 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v2
-; SI-NEXT: v_readfirstlane_b32 s6, v1
+; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s7, v0
+; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s8, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s6, 16
-; SI-NEXT: s_and_b32 s10, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s7, 16
-; SI-NEXT: s_and_b32 s12, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s29, 16
-; SI-NEXT: s_and_b32 s14, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s28, 16
-; SI-NEXT: s_and_b32 s40, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s41, s27, 16
-; SI-NEXT: s_and_b32 s42, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s43, s26, 16
-; SI-NEXT: s_and_b32 s44, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s45, s25, 16
-; SI-NEXT: s_and_b32 s46, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s47, s24, 16
-; SI-NEXT: s_and_b32 s56, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s23, 16
-; SI-NEXT: s_and_b32 s58, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s22, 16
-; SI-NEXT: s_and_b32 s60, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s61, s21, 16
-; SI-NEXT: s_and_b32 s62, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s63, s20, 16
-; SI-NEXT: s_and_b32 s72, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s73, s19, 16
-; SI-NEXT: s_and_b32 s74, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s75, s18, 16
-; SI-NEXT: s_and_b32 s76, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s77, s17, 16
-; SI-NEXT: s_and_b32 s78, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s79, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s63, s5, 16
+; SI-NEXT: s_lshr_b32 s62, s29, 16
+; SI-NEXT: s_lshr_b32 s61, s27, 16
+; SI-NEXT: s_lshr_b32 s60, s25, 16
+; SI-NEXT: s_lshr_b32 s59, s23, 16
+; SI-NEXT: s_lshr_b32 s58, s21, 16
+; SI-NEXT: s_lshr_b32 s57, s19, 16
+; SI-NEXT: s_lshr_b32 s56, s17, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[46:47], 0
; SI-NEXT: s_branch .LBB45_3
; SI-NEXT: .LBB45_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: s_mov_b64 s[46:47], -1
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr61
; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr63
; SI-NEXT: .LBB45_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB45_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v20, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v18, s22, 1.0
-; SI-NEXT: v_add_f32_e64 v16, s23, 1.0
-; SI-NEXT: v_add_f32_e64 v14, s24, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v15, s5, 1.0
+; SI-NEXT: v_add_f32_e64 v14, s4, 1.0
+; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
+; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
+; SI-NEXT: v_lshr_b64 v[16:17], v[14:15], 16
+; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
-; SI-NEXT: v_add_f32_e64 v8, s27, 1.0
-; SI-NEXT: v_add_f32_e64 v6, s28, 1.0
-; SI-NEXT: v_add_f32_e64 v4, s29, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s7, 1.0
-; SI-NEXT: v_add_f32_e64 v0, s6, 1.0
-; SI-NEXT: v_add_f32_e64 v22, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v24, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v26, s18, 1.0
-; SI-NEXT: v_add_f32_e64 v28, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v30, s16, 1.0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_lshr_b64 v[17:18], v[12:13], 16
+; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
+; SI-NEXT: v_lshr_b64 v[18:19], v[10:11], 16
+; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
+; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
+; SI-NEXT: v_lshr_b64 v[19:20], v[8:9], 16
+; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
+; SI-NEXT: v_lshr_b64 v[20:21], v[6:7], 16
+; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
+; SI-NEXT: v_lshr_b64 v[21:22], v[4:5], 16
+; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
+; SI-NEXT: v_lshr_b64 v[22:23], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
; SI-NEXT: s_branch .LBB45_6
; SI-NEXT: .LBB45_5:
-; SI-NEXT: v_mov_b32_e32 v30, s79
-; SI-NEXT: v_mov_b32_e32 v31, s78
-; SI-NEXT: v_mov_b32_e32 v28, s77
-; SI-NEXT: v_mov_b32_e32 v29, s76
-; SI-NEXT: v_mov_b32_e32 v26, s75
-; SI-NEXT: v_mov_b32_e32 v27, s74
-; SI-NEXT: v_mov_b32_e32 v24, s73
-; SI-NEXT: v_mov_b32_e32 v25, s72
-; SI-NEXT: v_mov_b32_e32 v22, s63
-; SI-NEXT: v_mov_b32_e32 v23, s62
-; SI-NEXT: v_mov_b32_e32 v20, s61
-; SI-NEXT: v_mov_b32_e32 v21, s60
-; SI-NEXT: v_mov_b32_e32 v18, s59
-; SI-NEXT: v_mov_b32_e32 v19, s58
-; SI-NEXT: v_mov_b32_e32 v16, s57
-; SI-NEXT: v_mov_b32_e32 v17, s56
-; SI-NEXT: v_mov_b32_e32 v14, s47
-; SI-NEXT: v_mov_b32_e32 v15, s46
-; SI-NEXT: v_mov_b32_e32 v12, s45
-; SI-NEXT: v_mov_b32_e32 v13, s44
-; SI-NEXT: v_mov_b32_e32 v10, s43
-; SI-NEXT: v_mov_b32_e32 v11, s42
-; SI-NEXT: v_mov_b32_e32 v8, s41
-; SI-NEXT: v_mov_b32_e32 v9, s40
-; SI-NEXT: v_mov_b32_e32 v6, s15
-; SI-NEXT: v_mov_b32_e32 v7, s14
-; SI-NEXT: v_mov_b32_e32 v4, s13
-; SI-NEXT: v_mov_b32_e32 v5, s12
-; SI-NEXT: v_mov_b32_e32 v2, s11
-; SI-NEXT: v_mov_b32_e32 v3, s10
-; SI-NEXT: v_mov_b32_e32 v0, s9
-; SI-NEXT: v_mov_b32_e32 v1, s8
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v15, s5
+; SI-NEXT: v_mov_b32_e32 v32, s56
+; SI-NEXT: v_mov_b32_e32 v31, s57
+; SI-NEXT: v_mov_b32_e32 v30, s58
+; SI-NEXT: v_mov_b32_e32 v29, s59
+; SI-NEXT: v_mov_b32_e32 v28, s60
+; SI-NEXT: v_mov_b32_e32 v27, s61
+; SI-NEXT: v_mov_b32_e32 v26, s62
+; SI-NEXT: v_mov_b32_e32 v25, s63
+; SI-NEXT: v_mov_b32_e32 v16, s6
+; SI-NEXT: v_mov_b32_e32 v17, s8
+; SI-NEXT: v_mov_b32_e32 v18, s10
+; SI-NEXT: v_mov_b32_e32 v19, s12
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v21, s40
+; SI-NEXT: v_mov_b32_e32 v22, s42
+; SI-NEXT: v_mov_b32_e32 v23, s44
; SI-NEXT: .LBB45_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[30:31], 16
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[28:29], 16
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[26:27], 16
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[24:25], 16
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v5
-; SI-NEXT: v_lshr_b64 v[4:5], v[22:23], 16
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[20:21], 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[18:19], 16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[16:17], 16
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[14:15], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v11
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[32:33], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[38:39], 16
-; SI-NEXT: v_lshr_b64 v[14:15], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[50:51], 16
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v23
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v32
+; SI-NEXT: v_or_b32_e32 v2, v2, v22
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v31
+; SI-NEXT: v_or_b32_e32 v4, v4, v21
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v30
+; SI-NEXT: v_or_b32_e32 v6, v6, v20
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v29
+; SI-NEXT: v_or_b32_e32 v8, v8, v19
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v28
+; SI-NEXT: v_or_b32_e32 v10, v10, v18
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v27
+; SI-NEXT: v_or_b32_e32 v12, v12, v17
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v26
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
+; SI-NEXT: v_or_b32_e32 v1, v1, v23
+; SI-NEXT: v_or_b32_e32 v3, v3, v22
+; SI-NEXT: v_or_b32_e32 v5, v5, v21
+; SI-NEXT: v_or_b32_e32 v7, v7, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16f32_to_v32bf16_scalar:
@@ -20617,71 +20288,55 @@ define <16 x float> @bitcast_v32bf16_to_v16f32(<32 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32bf16_to_v16f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v32, v15
+; SI-NEXT: v_mov_b32_e32 v17, v14
+; SI-NEXT: v_mov_b32_e32 v18, v13
+; SI-NEXT: v_mov_b32_e32 v19, v12
+; SI-NEXT: v_mov_b32_e32 v20, v11
+; SI-NEXT: v_mov_b32_e32 v21, v10
+; SI-NEXT: v_mov_b32_e32 v22, v9
+; SI-NEXT: v_mov_b32_e32 v23, v8
+; SI-NEXT: v_mov_b32_e32 v24, v7
+; SI-NEXT: v_mov_b32_e32 v25, v6
+; SI-NEXT: v_mov_b32_e32 v26, v5
+; SI-NEXT: v_mov_b32_e32 v27, v4
+; SI-NEXT: v_mov_b32_e32 v28, v3
+; SI-NEXT: v_mov_b32_e32 v29, v2
+; SI-NEXT: v_mov_b32_e32 v30, v1
+; SI-NEXT: v_mov_b32_e32 v31, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v31
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v15
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -20693,165 +20348,149 @@ define <16 x float> @bitcast_v32bf16_to_v16f32(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB46_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v52
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v16
-; SI-NEXT: v_alignbit_b32 v0, v0, v55, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v53, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v51, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v49, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v39, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v37, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v35, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v33, 16
-; SI-NEXT: v_alignbit_b32 v8, v8, v31, 16
-; SI-NEXT: v_alignbit_b32 v9, v9, v29, 16
-; SI-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; SI-NEXT: v_alignbit_b32 v11, v11, v25, 16
-; SI-NEXT: v_alignbit_b32 v12, v12, v23, 16
-; SI-NEXT: v_alignbit_b32 v13, v13, v21, 16
-; SI-NEXT: v_alignbit_b32 v14, v14, v19, 16
-; SI-NEXT: v_alignbit_b32 v15, v15, v17, 16
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v31
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v30
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v29
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v28
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v27
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v26
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v24
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v22
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v20
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v18
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v32
+; SI-NEXT: v_or_b32_e32 v0, v0, v55
+; SI-NEXT: v_or_b32_e32 v1, v1, v54
+; SI-NEXT: v_or_b32_e32 v2, v2, v53
+; SI-NEXT: v_or_b32_e32 v3, v3, v52
+; SI-NEXT: v_or_b32_e32 v4, v4, v51
+; SI-NEXT: v_or_b32_e32 v5, v5, v50
+; SI-NEXT: v_or_b32_e32 v6, v6, v49
+; SI-NEXT: v_or_b32_e32 v7, v7, v48
+; SI-NEXT: v_or_b32_e32 v8, v8, v39
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v37
+; SI-NEXT: v_or_b32_e32 v11, v11, v36
+; SI-NEXT: v_or_b32_e32 v12, v12, v35
+; SI-NEXT: v_or_b32_e32 v13, v13, v34
+; SI-NEXT: v_or_b32_e32 v14, v14, v33
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr52
; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB46_2
; SI-NEXT: .LBB46_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v52
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v31
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v55
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v54
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v53
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v28
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v52
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v27
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v36
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v26
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v50
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v34
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v25
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v49
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v32
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v48
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v30
; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v23
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v39
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v26
; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v21
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v37
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v24
; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v20
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v36
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v22
; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v35
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v20
; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v18
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v34
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v18
; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v33
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v32
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
@@ -22032,251 +21671,226 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-LABEL: bitcast_v32bf16_to_v16f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v2
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s14
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
+; SI-NEXT: v_writelane_b32 v18, s36, 0
+; SI-NEXT: v_writelane_b32 v18, s37, 1
+; SI-NEXT: v_writelane_b32 v18, s38, 2
+; SI-NEXT: v_writelane_b32 v18, s39, 3
+; SI-NEXT: v_writelane_b32 v18, s48, 4
+; SI-NEXT: v_writelane_b32 v18, s49, 5
+; SI-NEXT: v_writelane_b32 v18, s50, 6
+; SI-NEXT: v_writelane_b32 v18, s51, 7
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s8, s29, 16
+; SI-NEXT: s_lshr_b32 s11, s28, 16
+; SI-NEXT: s_lshr_b32 s12, s27, 16
+; SI-NEXT: s_lshr_b32 s13, s26, 16
+; SI-NEXT: s_lshr_b32 s14, s25, 16
+; SI-NEXT: s_lshr_b32 s15, s24, 16
+; SI-NEXT: s_lshr_b32 s56, s23, 16
+; SI-NEXT: s_lshr_b32 s57, s22, 16
+; SI-NEXT: s_lshr_b32 s58, s21, 16
+; SI-NEXT: s_lshr_b32 s59, s20, 16
+; SI-NEXT: s_lshr_b32 s60, s19, 16
+; SI-NEXT: s_lshr_b32 s61, s18, 16
+; SI-NEXT: s_lshr_b32 s62, s17, 16
+; SI-NEXT: s_lshr_b32 s63, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s7, 16
+; SI-NEXT: s_lshr_b32 s9, s10, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v34
-; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v32
-; SI-NEXT: v_lshr_b64 v[2:3], v[38:39], 16
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v30
-; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
-; SI-NEXT: v_mov_b32_e32 v7, v38
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v26
-; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
-; SI-NEXT: v_mov_b32_e32 v8, v36
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v24
-; SI-NEXT: v_mov_b32_e32 v38, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[35:36], 16
-; SI-NEXT: v_mov_b32_e32 v9, v34
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v22
-; SI-NEXT: v_mov_b32_e32 v36, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v10, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v20
-; SI-NEXT: v_mov_b32_e32 v34, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[31:32], 16
-; SI-NEXT: v_mov_b32_e32 v11, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v44
-; SI-NEXT: v_mov_b32_e32 v32, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v12, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v43
-; SI-NEXT: v_mov_b32_e32 v30, v11
-; SI-NEXT: v_lshr_b64 v[10:11], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v13, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v18
-; SI-NEXT: v_mov_b32_e32 v28, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v14, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v42
-; SI-NEXT: v_mov_b32_e32 v26, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v15, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v41
-; SI-NEXT: v_mov_b32_e32 v24, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[21:22], 16
-; SI-NEXT: v_mov_b32_e32 v39, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v40
-; SI-NEXT: v_mov_b32_e32 v22, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v20, v39
-; SI-NEXT: v_mov_b32_e32 v39, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
-; SI-NEXT: v_mov_b32_e32 v49, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v18, v39
-; SI-NEXT: v_mov_b32_e32 v16, v49
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s63, 16
+; SI-NEXT: s_or_b32 s36, s4, s5
+; SI-NEXT: s_and_b32 s4, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_or_b32 s37, s4, s5
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_or_b32 s38, s4, s5
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_or_b32 s39, s4, s5
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_or_b32 s40, s4, s5
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_or_b32 s41, s4, s5
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_or_b32 s42, s4, s5
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_or_b32 s43, s4, s5
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_or_b32 s44, s4, s5
+; SI-NEXT: s_and_b32 s4, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s45, s4, s5
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s46, s4, s5
+; SI-NEXT: s_and_b32 s4, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s47, s4, s5
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_or_b32 s48, s4, s5
+; SI-NEXT: s_and_b32 s4, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_or_b32 s49, s4, s5
+; SI-NEXT: s_and_b32 s4, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s50, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB47_3
; SI-NEXT: .LBB47_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; SI-NEXT: .LBB47_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB47_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v16, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s63, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s62, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s4, v16
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v16
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v16
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_add_f32_e32 v7, s4, v16
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; SI-NEXT: v_add_f32_e32 v8, s4, v16
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v44
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
+; SI-NEXT: v_add_f32_e32 v9, s4, v16
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v43
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, s4, v16
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
+; SI-NEXT: v_add_f32_e32 v11, s4, v16
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v42
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, s4, v16
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v41
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, s4, v16
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v40
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v14, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, s4, v16
+; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v17, s5, v16
+; SI-NEXT: v_add_f32_e32 v15, s4, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v17
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB47_5: ; %end
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: .LBB47_6: ; %end
+; SI-NEXT: v_readlane_b32 s51, v18, 7
+; SI-NEXT: v_readlane_b32 s50, v18, 6
+; SI-NEXT: v_readlane_b32 s49, v18, 5
+; SI-NEXT: v_readlane_b32 s48, v18, 4
+; SI-NEXT: v_readlane_b32 s39, v18, 3
+; SI-NEXT: v_readlane_b32 s38, v18, 2
+; SI-NEXT: v_readlane_b32 s37, v18, 1
+; SI-NEXT: v_readlane_b32 s36, v18, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -34248,90 +33862,42 @@ define <32 x bfloat> @bitcast_v8i64_to_v32bf16(<8 x i64> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB64_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v21, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v24, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v27, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB64_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB64_4
@@ -34352,104 +33918,72 @@ define <32 x bfloat> @bitcast_v8i64_to_v32bf16(<8 x i64> %a, i32 %b) #0 {
; SI-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
; SI-NEXT: v_addc_u32_e32 v15, vcc, 0, v15, vcc
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v21, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v24, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v27, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB64_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v26
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v24
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v27
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; SI-NEXT: v_or_b32_e32 v2, v2, v24
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v30
+; SI-NEXT: v_or_b32_e32 v4, v4, v21
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v29
+; SI-NEXT: v_or_b32_e32 v6, v6, v20
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v28
+; SI-NEXT: v_or_b32_e32 v8, v8, v19
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v26
+; SI-NEXT: v_or_b32_e32 v10, v10, v18
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v25
+; SI-NEXT: v_or_b32_e32 v12, v12, v17
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v22
+; SI-NEXT: v_or_b32_e32 v1, v1, v27
+; SI-NEXT: v_or_b32_e32 v3, v3, v24
+; SI-NEXT: v_or_b32_e32 v5, v5, v21
+; SI-NEXT: v_or_b32_e32 v7, v7, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i64_to_v32bf16:
@@ -34565,198 +34099,150 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v2
-; SI-NEXT: v_readfirstlane_b32 s78, v1
+; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s79, v0
+; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s78, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s78, 16
-; SI-NEXT: s_and_b32 s9, s79, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s79, 16
-; SI-NEXT: s_and_b32 s11, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s29, 16
-; SI-NEXT: s_and_b32 s13, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s28, 16
-; SI-NEXT: s_and_b32 s15, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s27, 16
-; SI-NEXT: s_and_b32 s41, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s26, 16
-; SI-NEXT: s_and_b32 s43, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s25, 16
-; SI-NEXT: s_and_b32 s45, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s24, 16
-; SI-NEXT: s_and_b32 s47, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s23, 16
-; SI-NEXT: s_and_b32 s57, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s22, 16
-; SI-NEXT: s_and_b32 s59, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s21, 16
-; SI-NEXT: s_and_b32 s61, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s20, 16
-; SI-NEXT: s_and_b32 s63, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s19, 16
-; SI-NEXT: s_and_b32 s73, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s18, 16
-; SI-NEXT: s_and_b32 s75, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s17, 16
-; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s56, s5, 16
+; SI-NEXT: s_lshr_b32 s57, s29, 16
+; SI-NEXT: s_lshr_b32 s58, s27, 16
+; SI-NEXT: s_lshr_b32 s59, s25, 16
+; SI-NEXT: s_lshr_b32 s60, s23, 16
+; SI-NEXT: s_lshr_b32 s61, s21, 16
+; SI-NEXT: s_lshr_b32 s62, s19, 16
+; SI-NEXT: s_lshr_b32 s63, s17, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[46:47], 0
; SI-NEXT: s_branch .LBB65_3
; SI-NEXT: .LBB65_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: s_mov_b64 s[46:47], -1
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr63
; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr61
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: .LBB65_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB65_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_u32 s4, s16, 3
-; SI-NEXT: s_addc_u32 s5, s17, 0
-; SI-NEXT: s_add_u32 s16, s18, 3
-; SI-NEXT: s_addc_u32 s17, s19, 0
-; SI-NEXT: s_add_u32 s18, s20, 3
-; SI-NEXT: s_addc_u32 s19, s21, 0
-; SI-NEXT: s_add_u32 s20, s22, 3
-; SI-NEXT: s_addc_u32 s21, s23, 0
-; SI-NEXT: s_add_u32 s22, s24, 3
-; SI-NEXT: s_addc_u32 s23, s25, 0
-; SI-NEXT: s_add_u32 s24, s26, 3
-; SI-NEXT: s_addc_u32 s14, s27, 0
-; SI-NEXT: s_add_u32 s12, s28, 3
-; SI-NEXT: s_addc_u32 s10, s29, 0
-; SI-NEXT: s_add_u32 s8, s79, 3
-; SI-NEXT: s_addc_u32 s6, s78, 0
-; SI-NEXT: s_and_b32 s7, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s9, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_and_b32 s11, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s13, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s15, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_and_b32 s41, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s24, 16
-; SI-NEXT: s_and_b32 s43, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s23, 16
-; SI-NEXT: s_and_b32 s45, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s22, 16
-; SI-NEXT: s_and_b32 s47, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s21, 16
-; SI-NEXT: s_and_b32 s57, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s20, 16
-; SI-NEXT: s_and_b32 s59, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s19, 16
-; SI-NEXT: s_and_b32 s61, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s18, 16
-; SI-NEXT: s_and_b32 s63, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s17, 16
-; SI-NEXT: s_and_b32 s73, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s16, 16
-; SI-NEXT: s_and_b32 s75, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s5, 16
-; SI-NEXT: s_and_b32 s77, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s4, 16
+; SI-NEXT: s_add_u32 s4, s4, 3
+; SI-NEXT: s_addc_u32 s5, s5, 0
+; SI-NEXT: s_add_u32 s28, s28, 3
+; SI-NEXT: s_addc_u32 s29, s29, 0
+; SI-NEXT: s_add_u32 s26, s26, 3
+; SI-NEXT: s_addc_u32 s27, s27, 0
+; SI-NEXT: s_add_u32 s24, s24, 3
+; SI-NEXT: s_addc_u32 s25, s25, 0
+; SI-NEXT: s_add_u32 s22, s22, 3
+; SI-NEXT: s_addc_u32 s23, s23, 0
+; SI-NEXT: s_add_u32 s20, s20, 3
+; SI-NEXT: s_addc_u32 s21, s21, 0
+; SI-NEXT: s_add_u32 s18, s18, 3
+; SI-NEXT: s_addc_u32 s19, s19, 0
+; SI-NEXT: s_add_u32 s16, s16, 3
+; SI-NEXT: s_addc_u32 s17, s17, 0
+; SI-NEXT: s_lshr_b32 s56, s5, 16
+; SI-NEXT: s_lshr_b32 s57, s29, 16
+; SI-NEXT: s_lshr_b32 s58, s27, 16
+; SI-NEXT: s_lshr_b32 s59, s25, 16
+; SI-NEXT: s_lshr_b32 s60, s23, 16
+; SI-NEXT: s_lshr_b32 s61, s21, 16
+; SI-NEXT: s_lshr_b32 s62, s19, 16
+; SI-NEXT: s_lshr_b32 s63, s17, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
; SI-NEXT: .LBB65_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s74
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s73
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s72
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s63
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s61
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s60
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s59
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s58
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s57
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s56
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s47
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s46
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s45
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s44
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s43
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s42
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s41
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s40
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: s_and_b32 s7, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s44, 16
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_and_b32 s9, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s63, 16
+; SI-NEXT: s_or_b32 s9, s9, s11
+; SI-NEXT: s_and_b32 s11, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s42, 16
+; SI-NEXT: s_or_b32 s11, s11, s13
+; SI-NEXT: s_and_b32 s13, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s62, 16
+; SI-NEXT: s_or_b32 s13, s13, s15
+; SI-NEXT: s_and_b32 s15, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s40, 16
+; SI-NEXT: s_or_b32 s15, s15, s16
+; SI-NEXT: s_and_b32 s16, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s61, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_and_b32 s17, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_or_b32 s14, s17, s14
+; SI-NEXT: s_and_b32 s17, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s60, 16
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_and_b32 s18, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s12, s18, s12
+; SI-NEXT: s_and_b32 s18, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s59, 16
+; SI-NEXT: s_or_b32 s18, s18, s19
+; SI-NEXT: s_and_b32 s19, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s10, s19, s10
+; SI-NEXT: s_and_b32 s19, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s58, 16
+; SI-NEXT: s_or_b32 s19, s19, s20
+; SI-NEXT: s_and_b32 s20, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s8, s20, s8
+; SI-NEXT: s_and_b32 s20, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s57, 16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s56, 16
+; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: v_mov_b32_e32 v2, s11
+; SI-NEXT: v_mov_b32_e32 v3, s13
+; SI-NEXT: v_mov_b32_e32 v4, s15
+; SI-NEXT: v_mov_b32_e32 v5, s16
+; SI-NEXT: v_mov_b32_e32 v6, s14
+; SI-NEXT: v_mov_b32_e32 v7, s17
+; SI-NEXT: v_mov_b32_e32 v8, s12
+; SI-NEXT: v_mov_b32_e32 v9, s18
+; SI-NEXT: v_mov_b32_e32 v10, s10
+; SI-NEXT: v_mov_b32_e32 v11, s19
+; SI-NEXT: v_mov_b32_e32 v12, s8
+; SI-NEXT: v_mov_b32_e32 v13, s20
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i64_to_v32bf16_scalar:
@@ -34930,71 +34416,55 @@ define <8 x i64> @bitcast_v32bf16_to_v8i64(<32 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32bf16_to_v8i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v32, v15
+; SI-NEXT: v_mov_b32_e32 v17, v14
+; SI-NEXT: v_mov_b32_e32 v18, v13
+; SI-NEXT: v_mov_b32_e32 v19, v12
+; SI-NEXT: v_mov_b32_e32 v20, v11
+; SI-NEXT: v_mov_b32_e32 v21, v10
+; SI-NEXT: v_mov_b32_e32 v22, v9
+; SI-NEXT: v_mov_b32_e32 v23, v8
+; SI-NEXT: v_mov_b32_e32 v24, v7
+; SI-NEXT: v_mov_b32_e32 v25, v6
+; SI-NEXT: v_mov_b32_e32 v26, v5
+; SI-NEXT: v_mov_b32_e32 v27, v4
+; SI-NEXT: v_mov_b32_e32 v28, v3
+; SI-NEXT: v_mov_b32_e32 v29, v2
+; SI-NEXT: v_mov_b32_e32 v30, v1
+; SI-NEXT: v_mov_b32_e32 v31, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v31
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v15
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -35006,165 +34476,149 @@ define <8 x i64> @bitcast_v32bf16_to_v8i64(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB66_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v52
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v16
-; SI-NEXT: v_alignbit_b32 v0, v0, v55, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v53, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v51, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v49, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v39, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v37, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v35, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v33, 16
-; SI-NEXT: v_alignbit_b32 v8, v8, v31, 16
-; SI-NEXT: v_alignbit_b32 v9, v9, v29, 16
-; SI-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; SI-NEXT: v_alignbit_b32 v11, v11, v25, 16
-; SI-NEXT: v_alignbit_b32 v12, v12, v23, 16
-; SI-NEXT: v_alignbit_b32 v13, v13, v21, 16
-; SI-NEXT: v_alignbit_b32 v14, v14, v19, 16
-; SI-NEXT: v_alignbit_b32 v15, v15, v17, 16
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v31
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v30
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v29
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v28
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v27
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v26
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v24
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v22
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v20
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v18
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v32
+; SI-NEXT: v_or_b32_e32 v0, v0, v55
+; SI-NEXT: v_or_b32_e32 v1, v1, v54
+; SI-NEXT: v_or_b32_e32 v2, v2, v53
+; SI-NEXT: v_or_b32_e32 v3, v3, v52
+; SI-NEXT: v_or_b32_e32 v4, v4, v51
+; SI-NEXT: v_or_b32_e32 v5, v5, v50
+; SI-NEXT: v_or_b32_e32 v6, v6, v49
+; SI-NEXT: v_or_b32_e32 v7, v7, v48
+; SI-NEXT: v_or_b32_e32 v8, v8, v39
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v37
+; SI-NEXT: v_or_b32_e32 v11, v11, v36
+; SI-NEXT: v_or_b32_e32 v12, v12, v35
+; SI-NEXT: v_or_b32_e32 v13, v13, v34
+; SI-NEXT: v_or_b32_e32 v14, v14, v33
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr52
; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB66_2
; SI-NEXT: .LBB66_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v52
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v31
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v55
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v54
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v53
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v28
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v52
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v27
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v36
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v26
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v50
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v34
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v25
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v49
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v32
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v48
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v30
; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v23
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v39
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v26
; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v21
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v37
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v24
; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v20
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v36
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v22
; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v35
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v20
; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v18
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v34
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v18
; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v33
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v32
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
@@ -36345,251 +35799,226 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-LABEL: bitcast_v32bf16_to_v8i64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v2
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s14
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
+; SI-NEXT: v_writelane_b32 v18, s36, 0
+; SI-NEXT: v_writelane_b32 v18, s37, 1
+; SI-NEXT: v_writelane_b32 v18, s38, 2
+; SI-NEXT: v_writelane_b32 v18, s39, 3
+; SI-NEXT: v_writelane_b32 v18, s48, 4
+; SI-NEXT: v_writelane_b32 v18, s49, 5
+; SI-NEXT: v_writelane_b32 v18, s50, 6
+; SI-NEXT: v_writelane_b32 v18, s51, 7
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s8, s29, 16
+; SI-NEXT: s_lshr_b32 s11, s28, 16
+; SI-NEXT: s_lshr_b32 s12, s27, 16
+; SI-NEXT: s_lshr_b32 s13, s26, 16
+; SI-NEXT: s_lshr_b32 s14, s25, 16
+; SI-NEXT: s_lshr_b32 s15, s24, 16
+; SI-NEXT: s_lshr_b32 s56, s23, 16
+; SI-NEXT: s_lshr_b32 s57, s22, 16
+; SI-NEXT: s_lshr_b32 s58, s21, 16
+; SI-NEXT: s_lshr_b32 s59, s20, 16
+; SI-NEXT: s_lshr_b32 s60, s19, 16
+; SI-NEXT: s_lshr_b32 s61, s18, 16
+; SI-NEXT: s_lshr_b32 s62, s17, 16
+; SI-NEXT: s_lshr_b32 s63, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s7, 16
+; SI-NEXT: s_lshr_b32 s9, s10, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v34
-; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v32
-; SI-NEXT: v_lshr_b64 v[2:3], v[38:39], 16
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v30
-; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
-; SI-NEXT: v_mov_b32_e32 v7, v38
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v26
-; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
-; SI-NEXT: v_mov_b32_e32 v8, v36
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v24
-; SI-NEXT: v_mov_b32_e32 v38, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[35:36], 16
-; SI-NEXT: v_mov_b32_e32 v9, v34
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v22
-; SI-NEXT: v_mov_b32_e32 v36, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v10, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v20
-; SI-NEXT: v_mov_b32_e32 v34, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[31:32], 16
-; SI-NEXT: v_mov_b32_e32 v11, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v44
-; SI-NEXT: v_mov_b32_e32 v32, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v12, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v43
-; SI-NEXT: v_mov_b32_e32 v30, v11
-; SI-NEXT: v_lshr_b64 v[10:11], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v13, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v18
-; SI-NEXT: v_mov_b32_e32 v28, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v14, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v42
-; SI-NEXT: v_mov_b32_e32 v26, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v15, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v41
-; SI-NEXT: v_mov_b32_e32 v24, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[21:22], 16
-; SI-NEXT: v_mov_b32_e32 v39, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v40
-; SI-NEXT: v_mov_b32_e32 v22, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v20, v39
-; SI-NEXT: v_mov_b32_e32 v39, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
-; SI-NEXT: v_mov_b32_e32 v49, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v18, v39
-; SI-NEXT: v_mov_b32_e32 v16, v49
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s63, 16
+; SI-NEXT: s_or_b32 s36, s4, s5
+; SI-NEXT: s_and_b32 s4, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_or_b32 s37, s4, s5
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_or_b32 s38, s4, s5
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_or_b32 s39, s4, s5
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_or_b32 s40, s4, s5
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_or_b32 s41, s4, s5
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_or_b32 s42, s4, s5
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_or_b32 s43, s4, s5
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_or_b32 s44, s4, s5
+; SI-NEXT: s_and_b32 s4, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s45, s4, s5
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s46, s4, s5
+; SI-NEXT: s_and_b32 s4, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s47, s4, s5
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_or_b32 s48, s4, s5
+; SI-NEXT: s_and_b32 s4, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_or_b32 s49, s4, s5
+; SI-NEXT: s_and_b32 s4, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s50, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB67_3
; SI-NEXT: .LBB67_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; SI-NEXT: .LBB67_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB67_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v16, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s63, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s62, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s4, v16
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v16
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v16
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_add_f32_e32 v7, s4, v16
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; SI-NEXT: v_add_f32_e32 v8, s4, v16
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v44
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
+; SI-NEXT: v_add_f32_e32 v9, s4, v16
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v43
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, s4, v16
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
+; SI-NEXT: v_add_f32_e32 v11, s4, v16
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v42
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, s4, v16
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v41
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, s4, v16
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v40
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v14, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, s4, v16
+; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v17, s5, v16
+; SI-NEXT: v_add_f32_e32 v15, s4, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v17
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB67_5: ; %end
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: s_branch .LBB67_6
+; SI-NEXT: .LBB67_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: .LBB67_6: ; %end
+; SI-NEXT: v_readlane_b32 s51, v18, 7
+; SI-NEXT: v_readlane_b32 s50, v18, 6
+; SI-NEXT: v_readlane_b32 s49, v18, 5
+; SI-NEXT: v_readlane_b32 s48, v18, 4
+; SI-NEXT: v_readlane_b32 s39, v18, 3
+; SI-NEXT: v_readlane_b32 s38, v18, 2
+; SI-NEXT: v_readlane_b32 s37, v18, 1
+; SI-NEXT: v_readlane_b32 s36, v18, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -47634,82 +47063,42 @@ define <32 x bfloat> @bitcast_v8f64_to_v32bf16(<8 x double> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB80_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v22, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v25, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v28, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB80_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB80_4
@@ -47722,104 +47111,72 @@ define <32 x bfloat> @bitcast_v8f64_to_v32bf16(<8 x double> %a, i32 %b) #0 {
; SI-NEXT: v_add_f64 v[10:11], v[10:11], 1.0
; SI-NEXT: v_add_f64 v[14:15], v[14:15], 1.0
; SI-NEXT: v_add_f64 v[12:13], v[12:13], 1.0
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v12
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v11
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v9
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v7
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v6
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v4
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
+; SI-NEXT: v_alignbit_b32 v16, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v17, v13, v12, 16
+; SI-NEXT: v_alignbit_b32 v18, v11, v10, 16
+; SI-NEXT: v_alignbit_b32 v19, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v20, v7, v6, 16
+; SI-NEXT: v_alignbit_b32 v22, v5, v4, 16
+; SI-NEXT: v_alignbit_b32 v25, v3, v2, 16
+; SI-NEXT: v_alignbit_b32 v28, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: .LBB80_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v26
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v24
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v28
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v31
+; SI-NEXT: v_or_b32_e32 v2, v2, v25
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v30
+; SI-NEXT: v_or_b32_e32 v4, v4, v22
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v29
+; SI-NEXT: v_or_b32_e32 v6, v6, v20
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v27
+; SI-NEXT: v_or_b32_e32 v8, v8, v19
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v26
+; SI-NEXT: v_or_b32_e32 v10, v10, v18
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v24
+; SI-NEXT: v_or_b32_e32 v12, v12, v17
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; SI-NEXT: v_or_b32_e32 v1, v1, v28
+; SI-NEXT: v_or_b32_e32 v3, v3, v25
+; SI-NEXT: v_or_b32_e32 v5, v5, v22
+; SI-NEXT: v_or_b32_e32 v7, v7, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8f64_to_v32bf16:
@@ -47912,219 +47269,155 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s79, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s5, 16
-; SI-NEXT: s_and_b32 s77, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s4, 16
-; SI-NEXT: s_and_b32 s75, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s29, 16
-; SI-NEXT: s_and_b32 s73, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s28, 16
-; SI-NEXT: s_and_b32 s63, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s27, 16
-; SI-NEXT: s_and_b32 s61, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s26, 16
-; SI-NEXT: s_and_b32 s59, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s25, 16
-; SI-NEXT: s_and_b32 s57, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s24, 16
-; SI-NEXT: s_and_b32 s47, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s23, 16
-; SI-NEXT: s_and_b32 s45, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s22, 16
-; SI-NEXT: s_and_b32 s43, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s21, 16
-; SI-NEXT: s_and_b32 s41, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s20, 16
-; SI-NEXT: s_and_b32 s15, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s19, 16
-; SI-NEXT: s_and_b32 s13, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s18, 16
-; SI-NEXT: s_and_b32 s11, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s17, 16
-; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_lshr_b32 s63, s5, 16
+; SI-NEXT: s_lshr_b32 s62, s29, 16
+; SI-NEXT: s_lshr_b32 s61, s27, 16
+; SI-NEXT: s_lshr_b32 s60, s25, 16
+; SI-NEXT: s_lshr_b32 s59, s23, 16
+; SI-NEXT: s_lshr_b32 s58, s21, 16
+; SI-NEXT: s_lshr_b32 s57, s19, 16
+; SI-NEXT: s_lshr_b32 s56, s17, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[12:13], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[46:47], 0
; SI-NEXT: s_branch .LBB81_3
; SI-NEXT: .LBB81_2:
-; SI-NEXT: s_mov_b64 s[6:7], -1
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: s_mov_b64 s[46:47], -1
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr42
; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr79
; SI-NEXT: .LBB81_3: ; %Flow
-; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB81_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f64 v[19:20], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[14:15], s[24:25], 1.0
+; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
-; SI-NEXT: v_add_f64 v[2:3], s[4:5], 1.0
-; SI-NEXT: v_add_f64 v[6:7], s[28:29], 1.0
-; SI-NEXT: v_add_f64 v[23:24], s[20:21], 1.0
-; SI-NEXT: v_add_f64 v[27:28], s[18:19], 1.0
-; SI-NEXT: v_add_f64 v[30:31], s[16:17], 1.0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v20
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v24
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v28
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v27
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v31
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: v_lshr_b64 v[16:17], v[14:15], 16
+; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
+; SI-NEXT: v_lshr_b64 v[17:18], v[12:13], 16
+; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
+; SI-NEXT: v_lshr_b64 v[18:19], v[10:11], 16
+; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
+; SI-NEXT: v_lshr_b64 v[19:20], v[8:9], 16
+; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; SI-NEXT: v_lshr_b64 v[20:21], v[6:7], 16
+; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; SI-NEXT: v_lshr_b64 v[21:22], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[22:23], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
; SI-NEXT: s_branch .LBB81_6
; SI-NEXT: .LBB81_5:
-; SI-NEXT: v_mov_b32_e32 v1, s79
-; SI-NEXT: v_mov_b32_e32 v0, s78
-; SI-NEXT: v_mov_b32_e32 v3, s77
-; SI-NEXT: v_mov_b32_e32 v2, s76
-; SI-NEXT: v_mov_b32_e32 v5, s75
-; SI-NEXT: v_mov_b32_e32 v4, s74
-; SI-NEXT: v_mov_b32_e32 v7, s73
-; SI-NEXT: v_mov_b32_e32 v6, s72
-; SI-NEXT: v_mov_b32_e32 v9, s63
-; SI-NEXT: v_mov_b32_e32 v8, s62
-; SI-NEXT: v_mov_b32_e32 v11, s61
-; SI-NEXT: v_mov_b32_e32 v10, s60
-; SI-NEXT: v_mov_b32_e32 v13, s59
-; SI-NEXT: v_mov_b32_e32 v12, s58
-; SI-NEXT: v_mov_b32_e32 v15, s57
-; SI-NEXT: v_mov_b32_e32 v14, s56
-; SI-NEXT: v_mov_b32_e32 v17, s47
-; SI-NEXT: v_mov_b32_e32 v16, s46
-; SI-NEXT: v_mov_b32_e32 v18, s45
-; SI-NEXT: v_mov_b32_e32 v19, s44
-; SI-NEXT: v_mov_b32_e32 v21, s43
-; SI-NEXT: v_mov_b32_e32 v20, s42
-; SI-NEXT: v_mov_b32_e32 v22, s41
-; SI-NEXT: v_mov_b32_e32 v23, s40
-; SI-NEXT: v_mov_b32_e32 v25, s15
-; SI-NEXT: v_mov_b32_e32 v24, s14
-; SI-NEXT: v_mov_b32_e32 v26, s13
-; SI-NEXT: v_mov_b32_e32 v27, s12
-; SI-NEXT: v_mov_b32_e32 v29, s11
-; SI-NEXT: v_mov_b32_e32 v28, s10
-; SI-NEXT: v_mov_b32_e32 v31, s9
-; SI-NEXT: v_mov_b32_e32 v30, s8
+; SI-NEXT: v_mov_b32_e32 v15, s5
+; SI-NEXT: v_mov_b32_e32 v13, s29
+; SI-NEXT: v_mov_b32_e32 v11, s27
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v10, s26
+; SI-NEXT: v_mov_b32_e32 v12, s28
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v25, s63
+; SI-NEXT: v_mov_b32_e32 v26, s62
+; SI-NEXT: v_mov_b32_e32 v27, s61
+; SI-NEXT: v_mov_b32_e32 v28, s60
+; SI-NEXT: v_mov_b32_e32 v29, s59
+; SI-NEXT: v_mov_b32_e32 v30, s58
+; SI-NEXT: v_mov_b32_e32 v31, s57
+; SI-NEXT: v_mov_b32_e32 v32, s56
+; SI-NEXT: v_mov_b32_e32 v23, s44
+; SI-NEXT: v_mov_b32_e32 v22, s42
+; SI-NEXT: v_mov_b32_e32 v21, s40
+; SI-NEXT: v_mov_b32_e32 v20, s14
+; SI-NEXT: v_mov_b32_e32 v19, s12
+; SI-NEXT: v_mov_b32_e32 v18, s10
+; SI-NEXT: v_mov_b32_e32 v17, s8
+; SI-NEXT: v_mov_b32_e32 v16, s6
; SI-NEXT: .LBB81_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[30:31], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v26
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[28:29], 16
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[32:33], 16
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v22
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[24:25], 16
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v5
-; SI-NEXT: v_lshr_b64 v[4:5], v[26:27], 16
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v18
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[20:21], 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[22:23], 16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[16:17], 16
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[14:15], 16
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v11
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[18:19], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[38:39], 16
-; SI-NEXT: v_lshr_b64 v[14:15], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[50:51], 16
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v0, v0, v23
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v32
+; SI-NEXT: v_or_b32_e32 v2, v2, v22
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v31
+; SI-NEXT: v_or_b32_e32 v4, v4, v21
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v30
+; SI-NEXT: v_or_b32_e32 v6, v6, v20
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v29
+; SI-NEXT: v_or_b32_e32 v8, v8, v19
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v28
+; SI-NEXT: v_or_b32_e32 v10, v10, v18
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v27
+; SI-NEXT: v_or_b32_e32 v12, v12, v17
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v26
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
+; SI-NEXT: v_or_b32_e32 v1, v1, v23
+; SI-NEXT: v_or_b32_e32 v3, v3, v22
+; SI-NEXT: v_or_b32_e32 v5, v5, v21
+; SI-NEXT: v_or_b32_e32 v7, v7, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8f64_to_v32bf16_scalar:
@@ -48304,71 +47597,55 @@ define <8 x double> @bitcast_v32bf16_to_v8f64(<32 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32bf16_to_v8f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v32, v15
+; SI-NEXT: v_mov_b32_e32 v17, v14
+; SI-NEXT: v_mov_b32_e32 v18, v13
+; SI-NEXT: v_mov_b32_e32 v19, v12
+; SI-NEXT: v_mov_b32_e32 v20, v11
+; SI-NEXT: v_mov_b32_e32 v21, v10
+; SI-NEXT: v_mov_b32_e32 v22, v9
+; SI-NEXT: v_mov_b32_e32 v23, v8
+; SI-NEXT: v_mov_b32_e32 v24, v7
+; SI-NEXT: v_mov_b32_e32 v25, v6
+; SI-NEXT: v_mov_b32_e32 v26, v5
+; SI-NEXT: v_mov_b32_e32 v27, v4
+; SI-NEXT: v_mov_b32_e32 v28, v3
+; SI-NEXT: v_mov_b32_e32 v29, v2
+; SI-NEXT: v_mov_b32_e32 v30, v1
+; SI-NEXT: v_mov_b32_e32 v31, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v31
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v15
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v14
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v12
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v9
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -48380,165 +47657,149 @@ define <8 x double> @bitcast_v32bf16_to_v8f64(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB82_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v52
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v26
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v18
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v16
-; SI-NEXT: v_alignbit_b32 v0, v0, v55, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v53, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v51, 16
-; SI-NEXT: v_alignbit_b32 v3, v3, v49, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v39, 16
-; SI-NEXT: v_alignbit_b32 v5, v5, v37, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v35, 16
-; SI-NEXT: v_alignbit_b32 v7, v7, v33, 16
-; SI-NEXT: v_alignbit_b32 v8, v8, v31, 16
-; SI-NEXT: v_alignbit_b32 v9, v9, v29, 16
-; SI-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; SI-NEXT: v_alignbit_b32 v11, v11, v25, 16
-; SI-NEXT: v_alignbit_b32 v12, v12, v23, 16
-; SI-NEXT: v_alignbit_b32 v13, v13, v21, 16
-; SI-NEXT: v_alignbit_b32 v14, v14, v19, 16
-; SI-NEXT: v_alignbit_b32 v15, v15, v17, 16
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v31
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v30
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v29
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v28
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v27
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v26
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v24
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v23
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v22
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v21
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v20
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v19
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v18
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v32
+; SI-NEXT: v_or_b32_e32 v0, v0, v55
+; SI-NEXT: v_or_b32_e32 v1, v1, v54
+; SI-NEXT: v_or_b32_e32 v2, v2, v53
+; SI-NEXT: v_or_b32_e32 v3, v3, v52
+; SI-NEXT: v_or_b32_e32 v4, v4, v51
+; SI-NEXT: v_or_b32_e32 v5, v5, v50
+; SI-NEXT: v_or_b32_e32 v6, v6, v49
+; SI-NEXT: v_or_b32_e32 v7, v7, v48
+; SI-NEXT: v_or_b32_e32 v8, v8, v39
+; SI-NEXT: v_or_b32_e32 v9, v9, v38
+; SI-NEXT: v_or_b32_e32 v10, v10, v37
+; SI-NEXT: v_or_b32_e32 v11, v11, v36
+; SI-NEXT: v_or_b32_e32 v12, v12, v35
+; SI-NEXT: v_or_b32_e32 v13, v13, v34
+; SI-NEXT: v_or_b32_e32 v14, v14, v33
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr52
; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB82_2
; SI-NEXT: .LBB82_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v52
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v31
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v55
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v54
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v53
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_alignbit_b32 v2, v3, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v49
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v28
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v52
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v27
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v36
; SI-NEXT: v_alignbit_b32 v4, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v26
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v50
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v34
; SI-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v25
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v49
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v32
; SI-NEXT: v_alignbit_b32 v6, v7, v6, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v48
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v30
; SI-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v23
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v39
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
; SI-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v38
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v26
; SI-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v21
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v37
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v24
; SI-NEXT: v_alignbit_b32 v10, v11, v10, 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v20
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v36
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v22
; SI-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v35
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v20
; SI-NEXT: v_alignbit_b32 v12, v13, v12, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v18
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v34
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v18
; SI-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v33
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_alignbit_b32 v14, v15, v14, 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v32
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
@@ -49719,251 +48980,226 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-LABEL: bitcast_v32bf16_to_v8f64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v2
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s14
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
+; SI-NEXT: v_writelane_b32 v18, s36, 0
+; SI-NEXT: v_writelane_b32 v18, s37, 1
+; SI-NEXT: v_writelane_b32 v18, s38, 2
+; SI-NEXT: v_writelane_b32 v18, s39, 3
+; SI-NEXT: v_writelane_b32 v18, s48, 4
+; SI-NEXT: v_writelane_b32 v18, s49, 5
+; SI-NEXT: v_writelane_b32 v18, s50, 6
+; SI-NEXT: v_writelane_b32 v18, s51, 7
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s8, s29, 16
+; SI-NEXT: s_lshr_b32 s11, s28, 16
+; SI-NEXT: s_lshr_b32 s12, s27, 16
+; SI-NEXT: s_lshr_b32 s13, s26, 16
+; SI-NEXT: s_lshr_b32 s14, s25, 16
+; SI-NEXT: s_lshr_b32 s15, s24, 16
+; SI-NEXT: s_lshr_b32 s56, s23, 16
+; SI-NEXT: s_lshr_b32 s57, s22, 16
+; SI-NEXT: s_lshr_b32 s58, s21, 16
+; SI-NEXT: s_lshr_b32 s59, s20, 16
+; SI-NEXT: s_lshr_b32 s60, s19, 16
+; SI-NEXT: s_lshr_b32 s61, s18, 16
+; SI-NEXT: s_lshr_b32 s62, s17, 16
+; SI-NEXT: s_lshr_b32 s63, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s7, 16
+; SI-NEXT: s_lshr_b32 s9, s10, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v34
-; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v32
-; SI-NEXT: v_lshr_b64 v[2:3], v[38:39], 16
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v30
-; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v28
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
-; SI-NEXT: v_mov_b32_e32 v7, v38
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v26
-; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
-; SI-NEXT: v_mov_b32_e32 v8, v36
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v24
-; SI-NEXT: v_mov_b32_e32 v38, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[35:36], 16
-; SI-NEXT: v_mov_b32_e32 v9, v34
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v22
-; SI-NEXT: v_mov_b32_e32 v36, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v10, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v20
-; SI-NEXT: v_mov_b32_e32 v34, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[31:32], 16
-; SI-NEXT: v_mov_b32_e32 v11, v30
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v44
-; SI-NEXT: v_mov_b32_e32 v32, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v12, v28
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v43
-; SI-NEXT: v_mov_b32_e32 v30, v11
-; SI-NEXT: v_lshr_b64 v[10:11], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v13, v26
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v18
-; SI-NEXT: v_mov_b32_e32 v28, v12
-; SI-NEXT: v_lshr_b64 v[11:12], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v14, v24
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v42
-; SI-NEXT: v_mov_b32_e32 v26, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[23:24], 16
-; SI-NEXT: v_mov_b32_e32 v15, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v41
-; SI-NEXT: v_mov_b32_e32 v24, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[21:22], 16
-; SI-NEXT: v_mov_b32_e32 v39, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v40
-; SI-NEXT: v_mov_b32_e32 v22, v15
-; SI-NEXT: v_lshr_b64 v[14:15], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v20, v39
-; SI-NEXT: v_mov_b32_e32 v39, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
-; SI-NEXT: v_mov_b32_e32 v49, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v18, v39
-; SI-NEXT: v_mov_b32_e32 v16, v49
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s63, 16
+; SI-NEXT: s_or_b32 s36, s4, s5
+; SI-NEXT: s_and_b32 s4, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_or_b32 s37, s4, s5
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_or_b32 s38, s4, s5
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_or_b32 s39, s4, s5
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_or_b32 s40, s4, s5
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_or_b32 s41, s4, s5
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_or_b32 s42, s4, s5
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_or_b32 s43, s4, s5
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_or_b32 s44, s4, s5
+; SI-NEXT: s_and_b32 s4, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_or_b32 s45, s4, s5
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_or_b32 s46, s4, s5
+; SI-NEXT: s_and_b32 s4, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_or_b32 s47, s4, s5
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_or_b32 s48, s4, s5
+; SI-NEXT: s_and_b32 s4, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_or_b32 s49, s4, s5
+; SI-NEXT: s_and_b32 s4, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s50, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB83_3
; SI-NEXT: .LBB83_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; SI-NEXT: .LBB83_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB83_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v16, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s63, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
+; SI-NEXT: s_lshl_b32 s4, s62, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v16
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_add_f32_e32 v3, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_add_f32_e32 v3, s4, v16
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v4, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s4, v16
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v26
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_add_f32_e32 v5, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v16
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v35
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_add_f32_e32 v6, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v16
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: s_lshl_b32 s5, s56, 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v7, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_add_f32_e32 v7, s4, v16
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_add_f32_e32 v8, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; SI-NEXT: v_add_f32_e32 v8, s4, v16
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v44
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_add_f32_e32 v9, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
+; SI-NEXT: v_add_f32_e32 v9, s4, v16
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v43
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_add_f32_e32 v10, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, s4, v16
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v25
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v11, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
+; SI-NEXT: v_add_f32_e32 v11, s4, v16
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v42
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v12, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, s4, v16
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v41
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v13, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
+; SI-NEXT: v_add_f32_e32 v13, s4, v16
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v40
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v14, s5, v16
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, s4, v16
+; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v17, s5, v16
+; SI-NEXT: v_add_f32_e32 v15, s4, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v17
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB83_5: ; %end
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: s_branch .LBB83_6
+; SI-NEXT: .LBB83_5:
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: v_mov_b32_e32 v3, s39
+; SI-NEXT: v_mov_b32_e32 v4, s40
+; SI-NEXT: v_mov_b32_e32 v5, s41
+; SI-NEXT: v_mov_b32_e32 v6, s42
+; SI-NEXT: v_mov_b32_e32 v7, s43
+; SI-NEXT: v_mov_b32_e32 v8, s44
+; SI-NEXT: v_mov_b32_e32 v9, s45
+; SI-NEXT: v_mov_b32_e32 v10, s46
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v12, s48
+; SI-NEXT: v_mov_b32_e32 v13, s49
+; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v15, s51
+; SI-NEXT: .LBB83_6: ; %end
+; SI-NEXT: v_readlane_b32 s51, v18, 7
+; SI-NEXT: v_readlane_b32 s50, v18, 6
+; SI-NEXT: v_readlane_b32 s49, v18, 5
+; SI-NEXT: v_readlane_b32 s48, v18, 4
+; SI-NEXT: v_readlane_b32 s39, v18, 3
+; SI-NEXT: v_readlane_b32 s38, v18, 2
+; SI-NEXT: v_readlane_b32 s37, v18, 1
+; SI-NEXT: v_readlane_b32 s36, v18, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -60265,75 +59501,127 @@ define <32 x bfloat> @bitcast_v32i16_to_v32bf16(<32 x i16> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32i16_to_v32bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v1
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v32
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v27
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v26
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v24
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v23
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v21
+; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr39
; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr35
; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB92_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v15
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v36, v1, v42
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v34, v1, v44
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_or_b32_e32 v55, v0, v40
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v32, v1, v46
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_or_b32_e32 v51, v0, v41
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v29, v1, v56
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_or_b32_e32 v39, v0, v43
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v26, v1, v58
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: v_or_b32_e32 v35, v0, v45
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v23, v1, v60
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v33, v0, v47
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v21, v1, v62
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v30, v0, v57
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v16, v1, v63
+; SI-NEXT: v_or_b32_e32 v27, v0, v59
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v14
+; SI-NEXT: v_alignbit_b32 v37, v36, v40, 16
+; SI-NEXT: v_alignbit_b32 v38, v34, v41, 16
+; SI-NEXT: v_alignbit_b32 v48, v32, v43, 16
+; SI-NEXT: v_alignbit_b32 v49, v29, v45, 16
+; SI-NEXT: v_alignbit_b32 v50, v26, v47, 16
+; SI-NEXT: v_alignbit_b32 v52, v23, v57, 16
+; SI-NEXT: v_alignbit_b32 v53, v21, v59, 16
+; SI-NEXT: v_alignbit_b32 v54, v16, v61, 16
+; SI-NEXT: v_or_b32_e32 v24, v0, v61
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
@@ -60350,197 +59638,198 @@ define <32 x bfloat> @bitcast_v32i16_to_v32bf16(<32 x i16> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: .LBB92_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB92_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v3
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v15, v31, v15
+; SI-NEXT: v_or_b32_e32 v14, v61, v14
; SI-NEXT: s_mov_b32 s6, 0x30000
-; SI-NEXT: v_or_b32_e32 v14, v30, v14
-; SI-NEXT: v_or_b32_e32 v13, v29, v13
-; SI-NEXT: v_or_b32_e32 v12, v28, v12
-; SI-NEXT: v_or_b32_e32 v11, v27, v11
-; SI-NEXT: v_or_b32_e32 v10, v26, v10
-; SI-NEXT: v_or_b32_e32 v9, v25, v9
-; SI-NEXT: v_or_b32_e32 v8, v24, v8
-; SI-NEXT: v_or_b32_e32 v7, v23, v7
-; SI-NEXT: v_or_b32_e32 v6, v22, v6
-; SI-NEXT: v_or_b32_e32 v5, v21, v5
-; SI-NEXT: v_or_b32_e32 v4, v20, v4
-; SI-NEXT: v_or_b32_e32 v3, v19, v3
-; SI-NEXT: v_or_b32_e32 v2, v18, v2
-; SI-NEXT: v_or_b32_e32 v1, v17, v1
-; SI-NEXT: v_or_b32_e32 v0, v16, v0
-; SI-NEXT: v_add_i32_e32 v15, vcc, 0x30000, v15
-; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT: v_add_i32_e32 v13, vcc, s6, v13
-; SI-NEXT: v_add_i32_e32 v12, vcc, s6, v12
-; SI-NEXT: v_add_i32_e32 v11, vcc, s6, v11
-; SI-NEXT: v_add_i32_e32 v10, vcc, s6, v10
-; SI-NEXT: v_add_i32_e32 v9, vcc, s6, v9
-; SI-NEXT: v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT: v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v4
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v0
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v1
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v2
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v3
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v4
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v5
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v6
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v9
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v10
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v11
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v14
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v15
+; SI-NEXT: v_or_b32_e32 v12, v59, v12
+; SI-NEXT: v_or_b32_e32 v10, v57, v10
+; SI-NEXT: v_or_b32_e32 v8, v47, v8
+; SI-NEXT: v_or_b32_e32 v6, v45, v6
+; SI-NEXT: v_or_b32_e32 v4, v43, v4
+; SI-NEXT: v_or_b32_e32 v2, v41, v2
+; SI-NEXT: v_or_b32_e32 v0, v40, v0
+; SI-NEXT: v_add_i32_e32 v24, vcc, 0x30000, v14
+; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v15
+; SI-NEXT: v_add_i32_e32 v27, vcc, s6, v12
+; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v13
+; SI-NEXT: v_add_i32_e32 v30, vcc, s6, v10
+; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v11
+; SI-NEXT: v_add_i32_e32 v33, vcc, s6, v8
+; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v9
+; SI-NEXT: v_add_i32_e32 v35, vcc, s6, v6
+; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v7
+; SI-NEXT: v_add_i32_e32 v39, vcc, s6, v4
+; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v5
+; SI-NEXT: v_add_i32_e32 v51, vcc, s6, v2
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v3
+; SI-NEXT: v_add_i32_e32 v55, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v14, v63, v14
+; SI-NEXT: v_or_b32_e32 v12, v62, v12
+; SI-NEXT: v_or_b32_e32 v10, v60, v10
+; SI-NEXT: v_or_b32_e32 v8, v58, v8
+; SI-NEXT: v_or_b32_e32 v6, v56, v6
+; SI-NEXT: v_or_b32_e32 v4, v46, v4
+; SI-NEXT: v_or_b32_e32 v2, v44, v2
+; SI-NEXT: v_or_b32_e32 v0, v42, v0
+; SI-NEXT: v_add_i32_e32 v16, vcc, s6, v14
+; SI-NEXT: v_add_i32_e32 v21, vcc, s6, v12
+; SI-NEXT: v_add_i32_e32 v23, vcc, s6, v10
+; SI-NEXT: v_add_i32_e32 v26, vcc, s6, v8
+; SI-NEXT: v_add_i32_e32 v29, vcc, s6, v6
+; SI-NEXT: v_add_i32_e32 v32, vcc, s6, v4
+; SI-NEXT: v_add_i32_e32 v34, vcc, s6, v2
+; SI-NEXT: v_add_i32_e32 v36, vcc, s6, v0
+; SI-NEXT: v_alignbit_b32 v37, v36, v55, 16
+; SI-NEXT: v_alignbit_b32 v38, v34, v51, 16
+; SI-NEXT: v_alignbit_b32 v48, v32, v39, 16
+; SI-NEXT: v_alignbit_b32 v49, v29, v35, 16
+; SI-NEXT: v_alignbit_b32 v50, v26, v33, 16
+; SI-NEXT: v_alignbit_b32 v52, v23, v30, 16
+; SI-NEXT: v_alignbit_b32 v53, v21, v27, 16
+; SI-NEXT: v_alignbit_b32 v54, v16, v24, 16
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v36
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v34
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
; SI-NEXT: .LBB92_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v17
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v33
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v18
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v35
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v20
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v37
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v22
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v39
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v49
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v26
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v51
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v28
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v53
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v30
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v54
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v55
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; SI-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: bitcast_v32i16_to_v32bf16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; VI-NEXT: s_cbranch_execz .LBB92_2
-; VI-NEXT: ; %bb.1: ; %cmp.true
-; VI-NEXT: v_mov_b32_e32 v16, 3
-; VI-NEXT: v_add_u16_sdwa v19, v15, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_add_u16_e32 v15, 3, v15
-; VI-NEXT: v_or_b32_e32 v15, v15, v19
-; VI-NEXT: v_add_u16_sdwa v19, v14, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_add_u16_e32 v14, 3, v14
-; VI-NEXT: v_or_b32_e32 v14, v14, v19
-; VI-NEXT: v_add_u16_sdwa v19, v13, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_add_u16_e32 v13, 3, v13
-; VI-NEXT: v_or_b32_e32 v13, v13, v19
-; VI-NEXT: v_add_u16_sdwa v19, v12, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_add_u16_e32 v12, 3, v12
-; VI-NEXT: v_or_b32_e32 v12, v12, v19
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v55
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v37
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v36
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v31
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v51
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v38
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v34
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v28
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v39
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v48
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v32
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v25
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v35
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v49
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v33
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v50
+; SI-NEXT: v_or_b32_e32 v8, v8, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v52
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v53
+; SI-NEXT: v_or_b32_e32 v12, v12, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v54
+; SI-NEXT: v_or_b32_e32 v14, v14, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: bitcast_v32i16_to_v32bf16:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
+; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; VI-NEXT: s_cbranch_execz .LBB92_2
+; VI-NEXT: ; %bb.1: ; %cmp.true
+; VI-NEXT: v_mov_b32_e32 v16, 3
+; VI-NEXT: v_add_u16_sdwa v19, v15, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_add_u16_e32 v15, 3, v15
+; VI-NEXT: v_or_b32_e32 v15, v15, v19
+; VI-NEXT: v_add_u16_sdwa v19, v14, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_add_u16_e32 v14, 3, v14
+; VI-NEXT: v_or_b32_e32 v14, v14, v19
+; VI-NEXT: v_add_u16_sdwa v19, v13, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_add_u16_e32 v13, 3, v13
+; VI-NEXT: v_or_b32_e32 v13, v13, v19
+; VI-NEXT: v_add_u16_sdwa v19, v12, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_add_u16_e32 v12, 3, v12
+; VI-NEXT: v_or_b32_e32 v12, v12, v19
; VI-NEXT: v_add_u16_sdwa v19, v11, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_u16_e32 v11, 3, v11
; VI-NEXT: v_or_b32_e32 v11, v11, v19
@@ -60661,189 +59950,209 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v17, s34, 0
-; SI-NEXT: v_writelane_b32 v17, s35, 1
-; SI-NEXT: v_writelane_b32 v17, s36, 2
-; SI-NEXT: v_writelane_b32 v17, s37, 3
-; SI-NEXT: v_writelane_b32 v17, s30, 4
-; SI-NEXT: v_writelane_b32 v17, s31, 5
-; SI-NEXT: v_readfirstlane_b32 s36, v1
-; SI-NEXT: v_readfirstlane_b32 s34, v0
-; SI-NEXT: s_lshr_b32 s30, s29, 16
-; SI-NEXT: s_lshr_b32 s4, s28, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
-; SI-NEXT: s_lshr_b32 vcc_lo, s26, 16
-; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s94, s24, 16
-; SI-NEXT: s_lshr_b32 s93, s23, 16
-; SI-NEXT: s_lshr_b32 s92, s22, 16
-; SI-NEXT: s_lshr_b32 s91, s21, 16
-; SI-NEXT: s_lshr_b32 s90, s20, 16
-; SI-NEXT: s_lshr_b32 s89, s19, 16
-; SI-NEXT: s_lshr_b32 s88, s18, 16
-; SI-NEXT: s_lshr_b32 s79, s17, 16
-; SI-NEXT: s_lshr_b32 s37, s16, 16
-; SI-NEXT: s_lshr_b32 s35, s36, 16
-; SI-NEXT: s_lshr_b32 s31, s34, 16
-; SI-NEXT: v_readfirstlane_b32 s5, v2
-; SI-NEXT: s_cmp_lg_u32 s5, 0
+; SI-NEXT: v_writelane_b32 v16, s34, 0
+; SI-NEXT: v_writelane_b32 v16, s35, 1
+; SI-NEXT: v_writelane_b32 v16, s30, 2
+; SI-NEXT: v_writelane_b32 v16, s31, 3
+; SI-NEXT: v_readfirstlane_b32 s63, v1
+; SI-NEXT: v_readfirstlane_b32 s59, v0
+; SI-NEXT: s_lshr_b32 s92, s29, 16
+; SI-NEXT: s_lshr_b32 s57, s28, 16
+; SI-NEXT: s_lshr_b32 s91, s27, 16
+; SI-NEXT: s_lshr_b32 s47, s26, 16
+; SI-NEXT: s_lshr_b32 s90, s25, 16
+; SI-NEXT: s_lshr_b32 s45, s24, 16
+; SI-NEXT: s_lshr_b32 s89, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s88, s21, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s20, 16
+; SI-NEXT: s_lshr_b32 s79, s19, 16
+; SI-NEXT: s_lshr_b32 s95, s18, 16
+; SI-NEXT: s_lshr_b32 s78, s17, 16
+; SI-NEXT: s_lshr_b32 s94, s16, 16
+; SI-NEXT: s_lshr_b32 s93, s63, 16
+; SI-NEXT: s_lshr_b32 s73, s59, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s74, s16, 16
-; SI-NEXT: s_lshl_b32 s78, s37, 16
-; SI-NEXT: s_lshl_b32 s72, s17, 16
-; SI-NEXT: s_lshl_b32 s77, s79, 16
-; SI-NEXT: s_lshl_b32 s62, s18, 16
-; SI-NEXT: s_lshl_b32 s76, s88, 16
-; SI-NEXT: s_lshl_b32 s60, s19, 16
-; SI-NEXT: s_lshl_b32 s75, s89, 16
-; SI-NEXT: s_lshl_b32 s58, s20, 16
-; SI-NEXT: s_lshl_b32 s73, s90, 16
-; SI-NEXT: s_lshl_b32 s56, s21, 16
-; SI-NEXT: s_lshl_b32 s63, s91, 16
-; SI-NEXT: s_lshl_b32 s46, s22, 16
-; SI-NEXT: s_lshl_b32 s61, s92, 16
-; SI-NEXT: s_lshl_b32 s44, s23, 16
-; SI-NEXT: s_lshl_b32 s59, s93, 16
-; SI-NEXT: s_lshl_b32 s42, s24, 16
-; SI-NEXT: s_lshl_b32 s57, s94, 16
-; SI-NEXT: s_lshl_b32 s43, s25, 16
-; SI-NEXT: s_lshl_b32 s47, s95, 16
-; SI-NEXT: s_lshl_b32 s14, s26, 16
-; SI-NEXT: s_lshl_b32 s45, vcc_lo, 16
-; SI-NEXT: s_lshl_b32 s12, s27, 16
-; SI-NEXT: s_lshl_b32 s40, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s15, s28, 16
-; SI-NEXT: s_mov_b32 s6, s4
-; SI-NEXT: s_lshl_b32 s41, s4, 16
-; SI-NEXT: s_lshl_b32 s9, s29, 16
-; SI-NEXT: s_lshl_b32 s13, s30, 16
-; SI-NEXT: s_lshl_b32 s10, s34, 16
-; SI-NEXT: s_lshl_b32 s11, s31, 16
-; SI-NEXT: s_lshl_b32 s7, s36, 16
-; SI-NEXT: s_lshl_b32 s8, s35, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s78, 16
+; SI-NEXT: s_or_b32 s75, s5, s7
+; SI-NEXT: s_and_b32 s5, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s79, 16
+; SI-NEXT: s_or_b32 s77, s5, s7
+; SI-NEXT: s_and_b32 s5, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s88, 16
+; SI-NEXT: s_or_b32 s31, s5, s7
+; SI-NEXT: s_and_b32 s7, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s89, 16
+; SI-NEXT: s_mov_b32 s5, s59
+; SI-NEXT: s_or_b32 s59, s7, s9
+; SI-NEXT: s_and_b32 s7, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s90, 16
+; SI-NEXT: s_or_b32 s61, s7, s9
+; SI-NEXT: s_and_b32 s7, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s91, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s74, s94, 16
+; SI-NEXT: s_or_b32 s43, s7, s9
+; SI-NEXT: s_and_b32 s7, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s92, 16
+; SI-NEXT: s_or_b32 s14, s4, s74
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s76, s95, 16
+; SI-NEXT: s_or_b32 s41, s7, s9
+; SI-NEXT: s_and_b32 s7, s63, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s93, 16
+; SI-NEXT: s_or_b32 s12, s4, s76
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s30, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s60, s45, 16
+; SI-NEXT: s_or_b32 s35, s7, s9
+; SI-NEXT: s_mov_b32 s7, s45
+; SI-NEXT: s_lshr_b64 s[44:45], s[74:75], 16
+; SI-NEXT: s_or_b32 s10, s4, s30
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s58, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s42, s47, 16
+; SI-NEXT: s_mov_b32 s45, s7
+; SI-NEXT: s_mov_b32 s7, s47
+; SI-NEXT: s_lshr_b64 s[46:47], s[76:77], 16
+; SI-NEXT: s_or_b32 s8, s4, s58
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s57, 16
+; SI-NEXT: s_mov_b32 s47, s7
+; SI-NEXT: s_mov_b32 s7, s57
+; SI-NEXT: s_lshr_b64 s[56:57], s[30:31], 16
+; SI-NEXT: s_or_b32 s6, s4, s60
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_mov_b32 s57, s7
+; SI-NEXT: s_mov_b32 s9, s59
+; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
+; SI-NEXT: s_mov_b32 s7, s61
+; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
+; SI-NEXT: s_or_b32 s4, s4, s42
+; SI-NEXT: s_mov_b32 s59, s5
+; SI-NEXT: s_mov_b32 s61, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[42:43], 16
+; SI-NEXT: s_and_b32 s42, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s34, s73, 16
+; SI-NEXT: s_mov_b32 s63, s61
+; SI-NEXT: s_or_b32 s42, s42, s40
+; SI-NEXT: s_mov_b32 s61, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[40:41], 16
+; SI-NEXT: s_and_b32 s40, s59, 0xffff
+; SI-NEXT: s_mov_b32 s15, s75
+; SI-NEXT: s_mov_b32 s13, s77
+; SI-NEXT: s_mov_b32 s11, s31
+; SI-NEXT: s_mov_b32 s5, s43
+; SI-NEXT: s_mov_b32 s43, s41
+; SI-NEXT: s_mov_b32 s73, s61
+; SI-NEXT: s_or_b32 s40, s40, s34
+; SI-NEXT: s_mov_b32 s41, s35
+; SI-NEXT: s_lshr_b64 s[74:75], s[34:35], 16
+; SI-NEXT: s_mov_b64 s[76:77], 0
; SI-NEXT: s_branch .LBB93_3
; SI-NEXT: .LBB93_2:
-; SI-NEXT: s_mov_b32 s6, s4
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: s_mov_b64 s[76:77], -1
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr44
; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: .LBB93_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_mov_b32 s4, s7
-; SI-NEXT: s_mov_b32 s5, s10
-; SI-NEXT: s_mov_b32 s7, s9
-; SI-NEXT: s_mov_b32 s9, s15
-; SI-NEXT: s_mov_b32 s10, s12
-; SI-NEXT: s_mov_b32 s12, s14
-; SI-NEXT: s_mov_b32 s14, s43
-; SI-NEXT: s_mov_b32 s15, s41
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s61, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s61, 1
; SI-NEXT: s_cbranch_scc1 .LBB93_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s36, s36, 3
-; SI-NEXT: s_and_b32 s4, s36, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
-; SI-NEXT: s_add_i32 s34, s34, 3
+; SI-NEXT: s_add_i32 s4, s59, 3
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s73, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s40, s4, 0x30000
+; SI-NEXT: s_add_i32 s4, s63, 3
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s31, 16
-; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: s_and_b32 s7, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s30, 16
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_add_i32 s41, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s57, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s29, s29, 3
+; SI-NEXT: s_add_i32 s42, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_add_i32 s43, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s47, 16
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_and_b32 s8, s27, 0xffff
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s91, 16
+; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s45, 16
+; SI-NEXT: s_add_i32 s25, s25, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s90, 16
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s8, s22, 0xffff
; SI-NEXT: s_lshl_b32 s9, vcc_hi, 16
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s10, vcc_lo, 16
-; SI-NEXT: s_add_i32 s25, s25, 3
+; SI-NEXT: s_and_b32 s9, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s89, 16
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s95, 16
-; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_and_b32 s10, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s11, vcc_lo, 16
+; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s94, 16
-; SI-NEXT: s_add_i32 s23, s23, 3
+; SI-NEXT: s_and_b32 s11, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s88, 16
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s93, 16
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_and_b32 s12, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s95, 16
+; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s92, 16
-; SI-NEXT: s_add_i32 s21, s21, 3
+; SI-NEXT: s_and_b32 s13, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s79, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_and_b32 s14, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s91, 16
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_and_b32 s15, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s90, 16
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_or_b32 s15, s20, s15
-; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s89, 16
-; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s19, s20, s19
-; SI-NEXT: s_and_b32 s18, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s88, 16
+; SI-NEXT: s_and_b32 s14, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s94, 16
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s18, s20, s18
-; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s79, 16
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_or_b32 s17, s20, s17
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s37, 16
-; SI-NEXT: s_or_b32 s16, s20, s16
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: s_and_b32 s15, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s78, 16
+; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
+; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
@@ -60852,115 +60161,93 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: s_add_i32 s19, s19, 0x30000
-; SI-NEXT: s_add_i32 s18, s18, 0x30000
-; SI-NEXT: s_add_i32 s17, s17, 0x30000
-; SI-NEXT: s_add_i32 s16, s16, 0x30000
-; SI-NEXT: s_and_b32 s78, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s16, 16
-; SI-NEXT: s_and_b32 s77, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s17, 16
-; SI-NEXT: s_and_b32 s76, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s18, 16
-; SI-NEXT: s_and_b32 s75, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s19, 16
-; SI-NEXT: s_and_b32 s73, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s15, 16
-; SI-NEXT: s_and_b32 s63, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s14, 16
-; SI-NEXT: s_and_b32 s61, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s13, 16
-; SI-NEXT: s_and_b32 s59, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s12, 16
-; SI-NEXT: s_and_b32 s57, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s11, 16
-; SI-NEXT: s_and_b32 s47, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s10, 16
-; SI-NEXT: s_and_b32 s45, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s9, 16
-; SI-NEXT: s_and_b32 s40, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s8, 16
-; SI-NEXT: s_and_b32 s15, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s6, 16
-; SI-NEXT: s_and_b32 s13, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s11, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[56:57], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 16
+; SI-NEXT: s_lshr_b32 s78, s15, 16
+; SI-NEXT: s_lshr_b32 s79, s13, 16
+; SI-NEXT: s_lshr_b32 s88, s11, 16
+; SI-NEXT: s_lshr_b32 s89, s9, 16
+; SI-NEXT: s_lshr_b32 s90, s7, 16
+; SI-NEXT: s_lshr_b32 s91, s5, 16
+; SI-NEXT: s_lshr_b32 s92, s43, 16
+; SI-NEXT: s_lshr_b32 s93, s41, 16
; SI-NEXT: .LBB93_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s78
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s77
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s72
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s76
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s60
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s73
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s58
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s63
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s56
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s61
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s46
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s59
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s44
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s57
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s42
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s47
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s45
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s40
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s5
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s4
-; SI-NEXT: v_readlane_b32 s30, v17, 4
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_readlane_b32 s31, v17, 5
-; SI-NEXT: v_readlane_b32 s37, v17, 3
-; SI-NEXT: v_readlane_b32 s36, v17, 2
-; SI-NEXT: v_readlane_b32 s35, v17, 1
-; SI-NEXT: v_readlane_b32 s34, v17, 0
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s44, 16
+; SI-NEXT: s_or_b32 s14, s14, s16
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s78, 16
+; SI-NEXT: s_or_b32 s15, s15, s16
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s46, 16
+; SI-NEXT: s_or_b32 s12, s12, s16
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s79, 16
+; SI-NEXT: s_or_b32 s13, s13, s16
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s56, 16
+; SI-NEXT: s_or_b32 s10, s10, s16
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s88, 16
+; SI-NEXT: s_or_b32 s11, s11, s16
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s58, 16
+; SI-NEXT: s_or_b32 s8, s8, s16
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s89, 16
+; SI-NEXT: s_or_b32 s9, s9, s16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s60, 16
+; SI-NEXT: s_or_b32 s6, s6, s16
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s90, 16
+; SI-NEXT: s_or_b32 s7, s7, s16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s62, 16
+; SI-NEXT: s_or_b32 s4, s4, s16
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s16, s91, 16
+; SI-NEXT: s_or_b32 s5, s5, s16
+; SI-NEXT: s_and_b32 s16, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s72, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_and_b32 s17, s43, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s92, 16
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_and_b32 s18, s40, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s74, 16
+; SI-NEXT: s_or_b32 s18, s18, s19
+; SI-NEXT: s_and_b32 s19, s41, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s93, 16
+; SI-NEXT: s_or_b32 s19, s19, s20
+; SI-NEXT: v_readlane_b32 s30, v16, 2
+; SI-NEXT: v_mov_b32_e32 v0, s14
+; SI-NEXT: v_mov_b32_e32 v1, s15
+; SI-NEXT: v_mov_b32_e32 v2, s12
+; SI-NEXT: v_mov_b32_e32 v3, s13
+; SI-NEXT: v_mov_b32_e32 v4, s10
+; SI-NEXT: v_mov_b32_e32 v5, s11
+; SI-NEXT: v_mov_b32_e32 v6, s8
+; SI-NEXT: v_mov_b32_e32 v7, s9
+; SI-NEXT: v_mov_b32_e32 v8, s6
+; SI-NEXT: v_mov_b32_e32 v9, s7
+; SI-NEXT: v_mov_b32_e32 v10, s4
+; SI-NEXT: v_mov_b32_e32 v11, s5
+; SI-NEXT: v_mov_b32_e32 v12, s16
+; SI-NEXT: v_mov_b32_e32 v13, s17
+; SI-NEXT: v_mov_b32_e32 v14, s18
+; SI-NEXT: v_mov_b32_e32 v15, s19
+; SI-NEXT: v_readlane_b32 s31, v16, 3
+; SI-NEXT: v_readlane_b32 s35, v16, 1
+; SI-NEXT: v_readlane_b32 s34, v16, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -61217,371 +60504,190 @@ define <32 x i16> @bitcast_v32bf16_to_v32i16(<32 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32bf16_to_v32i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v17
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB94_2
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v63
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v61
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v60
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v33
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v59
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v58
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v36
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v56
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v38
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v44
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v43
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v42
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v52
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v40
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v54
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: .LBB94_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB94_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v1
+; SI-NEXT: s_cbranch_execz .LBB94_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v23
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v58
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v56
-; SI-NEXT: v_alignbit_b32 v4, v4, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v46
-; SI-NEXT: v_alignbit_b32 v6, v6, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v44
-; SI-NEXT: v_alignbit_b32 v8, v8, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v42
-; SI-NEXT: v_alignbit_b32 v10, v10, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v40
-; SI-NEXT: v_alignbit_b32 v12, v12, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v41
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v54
-; SI-NEXT: v_alignbit_b32 v14, v14, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; SI-NEXT: v_alignbit_b32 v0, v32, v0, 16
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_alignbit_b32 v2, v32, v2, 16
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_alignbit_b32 v4, v32, v4, 16
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_alignbit_b32 v6, v32, v6, 16
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_alignbit_b32 v8, v32, v8, 16
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_alignbit_b32 v10, v32, v10, 16
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v52
-; SI-NEXT: v_alignbit_b32 v15, v16, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v53
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v13
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v50
-; SI-NEXT: v_alignbit_b32 v13, v17, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v51
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v11
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v48
-; SI-NEXT: v_alignbit_b32 v11, v18, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v49
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v9
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v38
-; SI-NEXT: v_alignbit_b32 v9, v19, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v39
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v7
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v36
-; SI-NEXT: v_alignbit_b32 v7, v20, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v5
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v33
-; SI-NEXT: v_alignbit_b32 v5, v21, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v34
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v3
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v23
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v27
-; SI-NEXT: v_alignbit_b32 v3, v22, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_alignbit_b32 v1, v23, v1, 16
+; SI-NEXT: v_alignbit_b32 v12, v32, v12, 16
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v24
+; SI-NEXT: v_alignbit_b32 v15, v17, v15, 16
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_alignbit_b32 v24, v1, v24, 16
-; SI-NEXT: v_alignbit_b32 v25, v3, v25, 16
-; SI-NEXT: v_alignbit_b32 v26, v5, v26, 16
-; SI-NEXT: v_alignbit_b32 v29, v7, v29, 16
-; SI-NEXT: v_alignbit_b32 v30, v9, v30, 16
-; SI-NEXT: v_alignbit_b32 v31, v11, v31, 16
-; SI-NEXT: v_alignbit_b32 v32, v13, v32, 16
-; SI-NEXT: v_alignbit_b32 v35, v15, v35, 16
-; SI-NEXT: .LBB94_4: ; %end
+; SI-NEXT: v_alignbit_b32 v13, v18, v13, 16
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_alignbit_b32 v11, v19, v11, 16
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_alignbit_b32 v9, v20, v9, 16
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_alignbit_b32 v7, v21, v7, 16
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_alignbit_b32 v5, v22, v5, 16
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_alignbit_b32 v3, v16, v3, 16
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_alignbit_b32 v1, v23, v1, 16
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; SI-NEXT: v_alignbit_b32 v14, v32, v14, 16
+; SI-NEXT: v_alignbit_b32 v31, v1, v31, 16
+; SI-NEXT: v_alignbit_b32 v30, v3, v30, 16
+; SI-NEXT: v_alignbit_b32 v29, v5, v29, 16
+; SI-NEXT: v_alignbit_b32 v28, v7, v28, 16
+; SI-NEXT: v_alignbit_b32 v27, v9, v27, 16
+; SI-NEXT: v_alignbit_b32 v26, v11, v26, 16
+; SI-NEXT: v_alignbit_b32 v25, v13, v25, 16
+; SI-NEXT: v_alignbit_b32 v24, v15, v24, 16
+; SI-NEXT: .LBB94_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v3, v3, v16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v29
+; SI-NEXT: v_or_b32_e32 v4, v4, v16
; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v22
+; SI-NEXT: v_or_b32_e32 v5, v5, v16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v28
+; SI-NEXT: v_or_b32_e32 v6, v6, v16
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; SI-NEXT: v_or_b32_e32 v7, v7, v16
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v27
+; SI-NEXT: v_or_b32_e32 v8, v8, v16
; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v16
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v26
+; SI-NEXT: v_or_b32_e32 v10, v10, v16
; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v16
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
+; SI-NEXT: v_or_b32_e32 v12, v12, v16
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v18
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: v_or_b32_e32 v13, v13, v16
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; SI-NEXT: v_or_b32_e32 v1, v1, v23
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v25
-; SI-NEXT: v_or_b32_e32 v3, v3, v22
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v26
-; SI-NEXT: v_or_b32_e32 v5, v5, v21
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v29
-; SI-NEXT: v_or_b32_e32 v7, v7, v20
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v30
-; SI-NEXT: v_or_b32_e32 v9, v9, v19
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v31
-; SI-NEXT: v_or_b32_e32 v11, v11, v18
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v32
-; SI-NEXT: v_or_b32_e32 v13, v13, v17
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v35
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v30
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_or_b32_e32 v0, v0, v24
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_or_b32_e32 v0, v0, v31
; SI-NEXT: v_or_b32_e32 v2, v2, v23
-; SI-NEXT: v_or_b32_e32 v4, v4, v22
-; SI-NEXT: v_or_b32_e32 v6, v6, v21
-; SI-NEXT: v_or_b32_e32 v8, v8, v20
-; SI-NEXT: v_or_b32_e32 v10, v10, v19
-; SI-NEXT: v_or_b32_e32 v12, v12, v18
-; SI-NEXT: v_or_b32_e32 v14, v14, v17
; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v32bf16_to_v32i16:
@@ -62692,388 +61798,252 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-LABEL: bitcast_v32bf16_to_v32i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v2
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s42
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s12, v1
+; SI-NEXT: v_readfirstlane_b32 s42, v0
+; SI-NEXT: s_lshr_b32 s13, s29, 16
+; SI-NEXT: s_lshr_b32 s14, s28, 16
+; SI-NEXT: s_lshr_b32 s11, s27, 16
+; SI-NEXT: s_lshr_b32 s40, s26, 16
+; SI-NEXT: s_lshr_b32 s10, s25, 16
+; SI-NEXT: s_lshr_b32 s43, s24, 16
+; SI-NEXT: s_lshr_b32 s9, s23, 16
+; SI-NEXT: s_lshr_b32 s44, s22, 16
+; SI-NEXT: s_lshr_b32 s8, s21, 16
+; SI-NEXT: s_lshr_b32 s45, s20, 16
+; SI-NEXT: s_lshr_b32 s7, s19, 16
+; SI-NEXT: s_lshr_b32 s46, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s47, s16, 16
+; SI-NEXT: s_lshr_b32 s41, s12, 16
+; SI-NEXT: s_lshr_b32 s15, s42, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_mov_b32_e32 v9, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v4
-; SI-NEXT: v_mov_b32_e32 v62, v23
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: v_mov_b32_e32 v59, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
-; SI-NEXT: v_mov_b32_e32 v56, v21
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; SI-NEXT: v_mov_b32_e32 v46, v5
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v8
-; SI-NEXT: v_mov_b32_e32 v45, v19
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v19
-; SI-NEXT: v_mov_b32_e32 v5, v15
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v15
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v20
-; SI-NEXT: v_mov_b32_e32 v61, v3
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v3
-; SI-NEXT: v_mov_b32_e32 v60, v1
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v12
-; SI-NEXT: v_mov_b32_e32 v58, v11
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v11
-; SI-NEXT: v_mov_b32_e32 v57, v7
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v44
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v34
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v48
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v55
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v30
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v18
; SI-NEXT: s_branch .LBB95_3
; SI-NEXT: .LBB95_2:
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v60, v1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_mov_b32_e32 v9, v17
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v62, v23
-; SI-NEXT: v_mov_b32_e32 v59, v13
-; SI-NEXT: v_mov_b32_e32 v56, v21
-; SI-NEXT: v_mov_b32_e32 v46, v5
-; SI-NEXT: v_mov_b32_e32 v45, v19
-; SI-NEXT: v_mov_b32_e32 v5, v15
-; SI-NEXT: v_mov_b32_e32 v61, v3
-; SI-NEXT: v_mov_b32_e32 v58, v11
-; SI-NEXT: v_mov_b32_e32 v57, v7
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: .LBB95_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB95_5
; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v0, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s47, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_lshr_b64 v[19:20], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s46, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_lshr_b64 v[20:21], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s45, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v7
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s44, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_lshr_b64 v[22:23], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s43, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_add_f32_e32 v11, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: v_lshr_b64 v[25:26], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[11:12], 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: s_lshl_b32 s14, s41, 16
+; SI-NEXT: v_add_f32_e32 v4, s14, v0
+; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s13, v0
+; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s11, v0
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s10, v0
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s9, v0
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[11:12], 16
+; SI-NEXT: v_add_f32_e32 v23, s12, v0
+; SI-NEXT: s_lshl_b32 s12, s29, 16
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s8, v0
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: v_add_f32_e32 v28, s12, v0
+; SI-NEXT: s_lshl_b32 s12, s27, 16
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s4, s42, 16
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: v_add_f32_e32 v30, s12, v0
+; SI-NEXT: s_lshl_b32 s11, s25, 16
+; SI-NEXT: s_lshl_b32 s10, s23, 16
+; SI-NEXT: s_lshl_b32 s9, s21, 16
+; SI-NEXT: s_lshl_b32 s8, s19, 16
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v4
+; SI-NEXT: s_lshl_b32 s7, s17, 16
+; SI-NEXT: v_add_f32_e32 v4, s6, v0
+; SI-NEXT: v_lshr_b64 v[15:16], v[23:24], 16
+; SI-NEXT: v_add_f32_e32 v32, s11, v0
+; SI-NEXT: v_add_f32_e32 v34, s10, v0
+; SI-NEXT: v_add_f32_e32 v36, s9, v0
+; SI-NEXT: v_add_f32_e32 v38, s8, v0
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v4
+; SI-NEXT: v_add_f32_e32 v50, s7, v0
+; SI-NEXT: v_add_f32_e32 v4, s5, v0
+; SI-NEXT: v_add_f32_e32 v11, s4, v0
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; SI-NEXT: v_lshr_b64 v[1:2], v[30:31], 16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; SI-NEXT: v_lshr_b64 v[3:4], v[32:33], 16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[34:35], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[11:12], 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; SI-NEXT: v_lshr_b64 v[7:8], v[36:37], 16
+; SI-NEXT: v_lshr_b64 v[12:13], v[50:51], 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v10
+; SI-NEXT: v_lshr_b64 v[9:10], v[38:39], 16
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshr_b64 v[40:41], v[11:12], 16
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[41:42], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[17:18], v[28:29], 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: v_lshr_b64 v[6:7], v[23:24], 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v59
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; SI-NEXT: v_lshr_b64 v[13:14], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v46
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_lshr_b64 v[23:24], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v38
-; SI-NEXT: v_lshr_b64 v[24:25], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_lshr_b64 v[25:26], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v57
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
-; SI-NEXT: v_lshr_b64 v[26:27], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v6
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v11
-; SI-NEXT: v_lshr_b64 v[27:28], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v5
-; SI-NEXT: v_lshr_b64 v[19:20], v[35:36], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v8
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
-; SI-NEXT: v_lshr_b64 v[15:16], v[28:29], 16
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v5
-; SI-NEXT: v_add_f32_e32 v50, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
-; SI-NEXT: v_lshr_b64 v[7:8], v[34:35], 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshr_b64 v[17:18], v[30:31], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v2
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v11
-; SI-NEXT: v_lshr_b64 v[11:12], v[32:33], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[3:4], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[39:40], v[6:7], 16
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v5
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
-; SI-NEXT: v_lshr_b64 v[40:41], v[18:19], 16
-; SI-NEXT: v_lshr_b64 v[21:22], v[37:38], 16
-; SI-NEXT: v_lshr_b64 v[52:53], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[41:42], v[10:11], 16
-; SI-NEXT: v_lshr_b64 v[53:54], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[42:43], v[16:17], 16
-; SI-NEXT: v_lshr_b64 v[54:55], v[20:21], 16
-; SI-NEXT: v_lshr_b64 v[43:44], v[14:15], 16
-; SI-NEXT: .LBB95_5: ; %end
+; SI-NEXT: v_lshr_b64 v[42:43], v[6:7], 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[43:44], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[10:11], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[52:53], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[14:15], 16
+; SI-NEXT: v_mov_b32_e32 v0, v40
+; SI-NEXT: v_mov_b32_e32 v2, v41
+; SI-NEXT: v_mov_b32_e32 v4, v42
+; SI-NEXT: v_mov_b32_e32 v6, v43
+; SI-NEXT: s_branch .LBB95_6
+; SI-NEXT: .LBB95_5:
+; SI-NEXT: v_mov_b32_e32 v51, s6
+; SI-NEXT: v_mov_b32_e32 v39, s7
+; SI-NEXT: v_mov_b32_e32 v37, s8
+; SI-NEXT: v_mov_b32_e32 v35, s9
+; SI-NEXT: v_mov_b32_e32 v33, s10
+; SI-NEXT: v_mov_b32_e32 v31, s11
+; SI-NEXT: v_mov_b32_e32 v29, s13
+; SI-NEXT: v_mov_b32_e32 v24, s41
+; SI-NEXT: v_mov_b32_e32 v48, s42
+; SI-NEXT: v_mov_b32_e32 v27, s28
+; SI-NEXT: v_mov_b32_e32 v26, s26
+; SI-NEXT: v_mov_b32_e32 v25, s24
+; SI-NEXT: v_mov_b32_e32 v22, s22
+; SI-NEXT: v_mov_b32_e32 v21, s20
+; SI-NEXT: v_mov_b32_e32 v20, s18
+; SI-NEXT: v_mov_b32_e32 v19, s16
+; SI-NEXT: v_mov_b32_e32 v0, s47
+; SI-NEXT: v_mov_b32_e32 v2, s46
+; SI-NEXT: v_mov_b32_e32 v4, s45
+; SI-NEXT: v_mov_b32_e32 v6, s44
+; SI-NEXT: v_mov_b32_e32 v54, s43
+; SI-NEXT: v_mov_b32_e32 v10, s40
+; SI-NEXT: v_mov_b32_e32 v52, s14
+; SI-NEXT: v_mov_b32_e32 v49, s15
+; SI-NEXT: v_mov_b32_e32 v12, s17
+; SI-NEXT: v_mov_b32_e32 v9, s19
+; SI-NEXT: v_mov_b32_e32 v7, s21
+; SI-NEXT: v_mov_b32_e32 v5, s23
+; SI-NEXT: v_mov_b32_e32 v3, s25
+; SI-NEXT: v_mov_b32_e32 v1, s27
+; SI-NEXT: v_mov_b32_e32 v17, s29
+; SI-NEXT: v_mov_b32_e32 v15, s12
+; SI-NEXT: .LBB95_6: ; %end
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v52
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v40
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v41
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v43
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v54
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v38
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v39
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v36
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v31
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_or_b32_e32 v4, v5, v4
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v37
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v5, v7, v5
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v51
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v53
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v49
-; SI-NEXT: v_or_b32_e32 v3, v3, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v42
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v21
-; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v23
-; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v35
-; SI-NEXT: v_or_b32_e32 v7, v7, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v24
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v39
+; SI-NEXT: v_or_b32_e32 v6, v7, v6
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v35
+; SI-NEXT: v_or_b32_e32 v1, v8, v1
+; SI-NEXT: v_or_b32_e32 v3, v9, v3
+; SI-NEXT: v_or_b32_e32 v7, v11, v7
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v54
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v19
-; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v25
-; SI-NEXT: v_or_b32_e32 v10, v10, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v33
-; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v33
+; SI-NEXT: v_or_b32_e32 v10, v11, v10
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v31
+; SI-NEXT: v_or_b32_e32 v9, v12, v9
+; SI-NEXT: v_or_b32_e32 v11, v13, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v52
; SI-NEXT: v_or_b32_e32 v12, v12, v13
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v29
; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v27
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v48
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v49
; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v29
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -72490,108 +71460,122 @@ define <32 x bfloat> @bitcast_v32f16_to_v32bf16(<32 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v32f16_to_v32bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v13
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v11
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v9
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v7
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v3
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v1
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr39
; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v54
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v41
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v42
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v43
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v44
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v45
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v46
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v47
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v56
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v57
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v58
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v59
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v60
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v61
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v62
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v15
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v63
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
+; SI-NEXT: v_or_b32_e32 v38, v1, v16
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v23
+; SI-NEXT: v_or_b32_e32 v39, v3, v16
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v22
+; SI-NEXT: v_or_b32_e32 v37, v5, v16
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; SI-NEXT: v_or_b32_e32 v36, v7, v16
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v20
+; SI-NEXT: v_or_b32_e32 v35, v9, v16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v19
+; SI-NEXT: v_or_b32_e32 v34, v11, v16
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v47
+; SI-NEXT: v_or_b32_e32 v33, v13, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v46
+; SI-NEXT: v_or_b32_e32 v31, v15, v16
+; SI-NEXT: v_or_b32_e32 v16, v0, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v45
+; SI-NEXT: v_or_b32_e32 v25, v0, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v44
+; SI-NEXT: v_or_b32_e32 v26, v0, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v43
+; SI-NEXT: v_or_b32_e32 v27, v0, v7
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v42
+; SI-NEXT: v_or_b32_e32 v28, v0, v9
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v41
+; SI-NEXT: v_or_b32_e32 v29, v0, v11
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v40
+; SI-NEXT: v_or_b32_e32 v30, v0, v13
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v14
+; SI-NEXT: v_alignbit_b32 v48, v38, v1, 16
+; SI-NEXT: v_alignbit_b32 v49, v39, v3, 16
+; SI-NEXT: v_alignbit_b32 v50, v37, v5, 16
+; SI-NEXT: v_alignbit_b32 v51, v36, v7, 16
+; SI-NEXT: v_alignbit_b32 v52, v35, v9, 16
+; SI-NEXT: v_alignbit_b32 v53, v34, v11, 16
+; SI-NEXT: v_alignbit_b32 v54, v33, v13, 16
+; SI-NEXT: v_alignbit_b32 v55, v31, v15, 16
+; SI-NEXT: v_or_b32_e32 v32, v0, v15
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
@@ -72608,236 +71592,212 @@ define <32 x bfloat> @bitcast_v32f16_to_v32bf16(<32 x half> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr40
; SI-NEXT: .LBB100_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v63
-; SI-NEXT: v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v17, v62
-; SI-NEXT: v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v15
-; SI-NEXT: v_cvt_f16_f32_e32 v15, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, v61
-; SI-NEXT: v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v50, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, v60
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v48, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, v59
-; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v38, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, v58
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v36, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v17
-; SI-NEXT: v_cvt_f16_f32_e32 v39, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v57
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v34, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v56
+; SI-NEXT: v_cvt_f32_f16_e32 v25, v46
; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v17, v54
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v32, v16
; SI-NEXT: v_cvt_f32_f16_e32 v16, v47
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v46
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v28, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v45
-; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v26, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v44
+; SI-NEXT: v_add_f32_e32 v25, 0x38000000, v25
+; SI-NEXT: v_cvt_f16_f32_e32 v25, v25
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT: v_cvt_f16_f32_e32 v27, v2
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v24, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v43
+; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v52, v18
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v25
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v26, v0
+; SI-NEXT: v_or_b32_e32 v25, v27, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v27, v44
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v16
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT: v_or_b32_e32 v16, v26, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v26, v45
+; SI-NEXT: v_cvt_f16_f32_e32 v28, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v27
+; SI-NEXT: v_cvt_f16_f32_e32 v27, v4
+; SI-NEXT: v_add_f32_e32 v26, 0x38000000, v26
+; SI-NEXT: v_cvt_f16_f32_e32 v26, v26
+; SI-NEXT: v_cvt_f32_f16_e32 v29, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v27
+; SI-NEXT: v_cvt_f32_f16_e32 v27, v43
+; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v26
+; SI-NEXT: v_or_b32_e32 v26, v28, v4
+; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v27
+; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v29
+; SI-NEXT: v_cvt_f16_f32_e32 v29, v27
+; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
+; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v29
+; SI-NEXT: v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT: v_or_b32_e32 v27, v28, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v31, v42
+; SI-NEXT: v_or_b32_e32 v28, v30, v8
+; SI-NEXT: v_cvt_f32_f16_e32 v30, v41
+; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT: v_add_f32_e32 v29, 0x38000000, v31
+; SI-NEXT: v_cvt_f16_f32_e32 v31, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v30
+; SI-NEXT: v_cvt_f16_f32_e32 v30, v10
+; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v30
+; SI-NEXT: v_cvt_f32_f16_e32 v30, v40
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v29
+; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT: v_or_b32_e32 v29, v31, v10
+; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v30
+; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT: v_cvt_f16_f32_e32 v31, v30
+; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v22, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v42
-; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_or_b32_e32 v30, v12, v54
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v17
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v31
+; SI-NEXT: v_or_b32_e32 v32, v14, v55
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v15
+; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v18
+; SI-NEXT: v_cvt_f16_f32_e32 v17, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v18, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_or_b32_e32 v31, v12, v14
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v19
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v18
+; SI-NEXT: v_cvt_f16_f32_e32 v19, v14
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT: v_cvt_f16_f32_e32 v20, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v16, v41
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT: v_or_b32_e32 v33, v13, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v20
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v19
+; SI-NEXT: v_or_b32_e32 v34, v11, v13
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v12
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v21
+; SI-NEXT: v_cvt_f16_f32_e32 v20, v11
+; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v12
+; SI-NEXT: v_cvt_f16_f32_e32 v21, v11
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v20
+; SI-NEXT: v_or_b32_e32 v35, v9, v11
+; SI-NEXT: v_cvt_f32_f16_e32 v11, v22
+; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v21
+; SI-NEXT: v_cvt_f16_f32_e32 v22, v11
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_or_b32_e32 v36, v7, v9
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v23
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; SI-NEXT: v_or_b32_e32 v37, v5, v9
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v23, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v24
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v24, v5
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v18, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v32
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v34
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v36
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
-; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v48
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v13
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v52
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v15
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v23
+; SI-NEXT: v_or_b32_e32 v39, v3, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
+; SI-NEXT: v_or_b32_e32 v38, v1, v3
+; SI-NEXT: v_alignbit_b32 v48, v38, v0, 16
+; SI-NEXT: v_alignbit_b32 v49, v39, v2, 16
+; SI-NEXT: v_alignbit_b32 v50, v37, v4, 16
+; SI-NEXT: v_alignbit_b32 v51, v36, v6, 16
+; SI-NEXT: v_alignbit_b32 v52, v35, v8, 16
+; SI-NEXT: v_alignbit_b32 v53, v34, v10, 16
+; SI-NEXT: v_alignbit_b32 v54, v33, v54, 16
+; SI-NEXT: v_alignbit_b32 v55, v31, v55, 16
; SI-NEXT: .LBB100_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v16
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v17
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v18
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v25
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v20
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v22
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v29
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v24
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v26
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v33
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v35
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v30
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v37
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v39
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v34
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v36
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v51
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v38
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v48
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v55
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v50
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v40
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v52
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v48
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v38
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v24
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v49
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v39
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v23
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v50
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v37
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v51
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v36
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v21
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v52
+; SI-NEXT: v_or_b32_e32 v8, v8, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v35
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v53
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v34
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v54
+; SI-NEXT: v_or_b32_e32 v12, v12, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v33
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v32
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v55
+; SI-NEXT: v_or_b32_e32 v14, v14, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -72990,335 +71950,355 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: v_writelane_b32 v40, s34, 0
; SI-NEXT: v_writelane_b32 v40, s35, 1
; SI-NEXT: v_writelane_b32 v40, s36, 2
-; SI-NEXT: v_writelane_b32 v40, s37, 3
-; SI-NEXT: v_writelane_b32 v40, s30, 4
-; SI-NEXT: v_writelane_b32 v40, s31, 5
-; SI-NEXT: v_readfirstlane_b32 s60, v1
-; SI-NEXT: v_readfirstlane_b32 s56, v0
-; SI-NEXT: s_lshr_b32 s43, s29, 16
-; SI-NEXT: s_lshr_b32 s42, s28, 16
-; SI-NEXT: s_lshr_b32 s41, s27, 16
-; SI-NEXT: s_lshr_b32 s40, s26, 16
-; SI-NEXT: s_lshr_b32 s15, s25, 16
-; SI-NEXT: s_lshr_b32 s14, s24, 16
-; SI-NEXT: s_lshr_b32 s13, s23, 16
+; SI-NEXT: v_writelane_b32 v40, s30, 3
+; SI-NEXT: v_writelane_b32 v40, s31, 4
+; SI-NEXT: v_readfirstlane_b32 s79, v1
+; SI-NEXT: v_readfirstlane_b32 s78, v0
+; SI-NEXT: s_lshr_b32 s93, s29, 16
+; SI-NEXT: s_lshr_b32 s95, s28, 16
+; SI-NEXT: s_lshr_b32 s94, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s92, s25, 16
+; SI-NEXT: s_lshr_b32 s15, s24, 16
+; SI-NEXT: s_lshr_b32 s91, s23, 16
; SI-NEXT: s_lshr_b32 s12, s22, 16
-; SI-NEXT: s_lshr_b32 s11, s21, 16
+; SI-NEXT: s_lshr_b32 s90, s21, 16
; SI-NEXT: s_lshr_b32 s10, s20, 16
-; SI-NEXT: s_lshr_b32 s9, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s7, s17, 16
-; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_lshr_b32 s63, s60, 16
-; SI-NEXT: s_lshr_b32 s59, s56, 16
+; SI-NEXT: s_lshr_b32 s89, s19, 16
+; SI-NEXT: s_lshr_b32 s6, s18, 16
+; SI-NEXT: s_lshr_b32 s88, s17, 16
+; SI-NEXT: s_lshr_b32 s34, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s79, 16
+; SI-NEXT: s_lshr_b32 s9, s78, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s44, s16, 16
-; SI-NEXT: s_lshl_b32 s45, s6, 16
-; SI-NEXT: s_lshl_b32 s46, s17, 16
-; SI-NEXT: s_lshl_b32 s47, s7, 16
-; SI-NEXT: s_lshl_b32 s57, s18, 16
-; SI-NEXT: s_lshl_b32 s58, s8, 16
-; SI-NEXT: s_lshl_b32 s61, s19, 16
-; SI-NEXT: s_lshl_b32 s62, s9, 16
-; SI-NEXT: s_lshl_b32 s72, s20, 16
-; SI-NEXT: s_lshl_b32 s73, s10, 16
-; SI-NEXT: s_lshl_b32 s74, s21, 16
-; SI-NEXT: s_lshl_b32 s75, s11, 16
-; SI-NEXT: s_lshl_b32 s76, s22, 16
-; SI-NEXT: s_lshl_b32 s77, s12, 16
-; SI-NEXT: s_lshl_b32 s78, s23, 16
-; SI-NEXT: s_lshl_b32 s79, s13, 16
-; SI-NEXT: s_lshl_b32 s88, s24, 16
-; SI-NEXT: s_lshl_b32 s89, s14, 16
-; SI-NEXT: s_lshl_b32 s90, s25, 16
-; SI-NEXT: s_lshl_b32 s91, s15, 16
-; SI-NEXT: s_lshl_b32 s92, s26, 16
-; SI-NEXT: s_lshl_b32 s93, s40, 16
-; SI-NEXT: s_lshl_b32 s94, s27, 16
-; SI-NEXT: s_lshl_b32 s95, s41, 16
-; SI-NEXT: s_lshl_b32 vcc_lo, s28, 16
-; SI-NEXT: s_lshl_b32 vcc_hi, s42, 16
-; SI-NEXT: s_lshl_b32 s30, s29, 16
-; SI-NEXT: s_lshl_b32 s31, s43, 16
-; SI-NEXT: s_lshl_b32 s34, s56, 16
-; SI-NEXT: s_lshl_b32 s35, s59, 16
-; SI-NEXT: s_lshl_b32 s36, s60, 16
-; SI-NEXT: s_lshl_b32 s37, s63, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s34, 16
+; SI-NEXT: s_or_b32 s14, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_mov_b32 s11, s6
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s30, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_mov_b32 s13, s10
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s31, s5, s10
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_mov_b32 s41, s12
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s35, s5, s12
+; SI-NEXT: s_and_b32 s5, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s15, 16
+; SI-NEXT: s_or_b32 s36, s5, s40
+; SI-NEXT: s_and_b32 s5, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s46, vcc_hi, 16
+; SI-NEXT: s_or_b32 s47, s5, s46
+; SI-NEXT: s_and_b32 s5, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s42, s95, 16
+; SI-NEXT: s_or_b32 s57, s5, s42
+; SI-NEXT: s_and_b32 s5, s78, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s9, 16
+; SI-NEXT: s_or_b32 s62, s5, s56
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s88, 16
+; SI-NEXT: s_or_b32 s5, s5, s7
+; SI-NEXT: s_mov_b32 s7, s9
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: s_mov_b32 s4, s14
+; SI-NEXT: s_mov_b32 s9, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s89, 16
+; SI-NEXT: s_or_b32 s7, s7, s14
+; SI-NEXT: s_mov_b32 s43, s15
+; SI-NEXT: s_lshr_b64 s[14:15], s[6:7], 16
+; SI-NEXT: s_mov_b32 s6, s11
+; SI-NEXT: s_mov_b32 s15, s43
+; SI-NEXT: s_and_b32 s11, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s90, 16
+; SI-NEXT: s_or_b32 s11, s11, s43
+; SI-NEXT: s_lshr_b64 s[44:45], s[10:11], 16
+; SI-NEXT: s_mov_b32 s10, s13
+; SI-NEXT: s_and_b32 s13, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s91, 16
+; SI-NEXT: s_or_b32 s13, s13, s43
+; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
+; SI-NEXT: s_mov_b32 s12, s41
+; SI-NEXT: s_and_b32 s41, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s92, 16
+; SI-NEXT: s_or_b32 s41, s41, s43
+; SI-NEXT: s_lshr_b64 s[60:61], s[40:41], 16
+; SI-NEXT: s_mov_b32 s40, s47
+; SI-NEXT: s_and_b32 s43, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s94, 16
+; SI-NEXT: s_mov_b32 s45, s57
+; SI-NEXT: s_or_b32 s47, s43, s47
+; SI-NEXT: s_and_b32 s43, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s57, s93, 16
+; SI-NEXT: s_or_b32 s43, s43, s57
+; SI-NEXT: s_and_b32 s57, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s61, vcc_lo, 16
+; SI-NEXT: s_or_b32 s57, s57, s61
+; SI-NEXT: s_mov_b32 s59, s62
+; SI-NEXT: s_lshr_b64 s[62:63], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[56:57], 16
+; SI-NEXT: s_mov_b64 s[76:77], 0
; SI-NEXT: s_branch .LBB101_3
; SI-NEXT: .LBB101_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[76:77], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr31
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr35
; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr45
; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: .LBB101_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s42, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s42, 1
; SI-NEXT: s_cbranch_scc1 .LBB101_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s60
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s59
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s42
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s40
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s56
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s28
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s10
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v16, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s43
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s29
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s41
-; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s27
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s26
-; SI-NEXT: v_cvt_f16_f32_e32 v32, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s15
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s24
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s13
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s23
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s12
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s11
-; SI-NEXT: v_cvt_f32_f16_e32 v20, s21
-; SI-NEXT: v_cvt_f32_f16_e32 v21, s10
-; SI-NEXT: v_cvt_f32_f16_e32 v22, s20
-; SI-NEXT: v_cvt_f32_f16_e32 v23, s9
-; SI-NEXT: v_cvt_f32_f16_e32 v24, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v25, s8
-; SI-NEXT: v_cvt_f32_f16_e32 v26, s18
-; SI-NEXT: v_cvt_f32_f16_e32 v27, s7
-; SI-NEXT: v_cvt_f32_f16_e32 v28, s17
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s6
-; SI-NEXT: v_cvt_f32_f16_e32 v30, s16
+; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s12
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s15
+; SI-NEXT: v_cvt_f32_f16_e32 v5, vcc_hi
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
-; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s95
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s9
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v5, vcc_lo
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v12, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v14, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s94
+; SI-NEXT: v_cvt_f16_f32_e32 v32, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s27
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s92
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v33, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v34, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v33
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s25
+; SI-NEXT: v_or_b32_e32 v11, v1, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s91
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s23
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v35, v3
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v34
+; SI-NEXT: v_or_b32_e32 v9, v5, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s90
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v35
+; SI-NEXT: v_or_b32_e32 v7, v3, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s21
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s89
+; SI-NEXT: v_cvt_f16_f32_e32 v36, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v37, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v36
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s19
+; SI-NEXT: v_or_b32_e32 v5, v1, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s88
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s17
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v39, v3
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v19
+; SI-NEXT: v_cvt_f16_f32_e32 v19, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v13, s93
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v37
+; SI-NEXT: v_or_b32_e32 v3, v15, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s29
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v39
+; SI-NEXT: v_or_b32_e32 v1, v19, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s79
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT: v_cvt_f16_f32_e32 v38, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v15
+; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v19
+; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v38
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s34
+; SI-NEXT: v_or_b32_e32 v13, v13, v19
+; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v32
+; SI-NEXT: v_cvt_f32_f16_e32 v18, s20
+; SI-NEXT: v_or_b32_e32 v15, v15, v19
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s22
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
+; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
+; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
+; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v55, v16, v0
+; SI-NEXT: v_or_b32_e32 v53, v17, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s24
+; SI-NEXT: v_or_b32_e32 v52, v18, v4
+; SI-NEXT: v_or_b32_e32 v49, v19, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s26
+; SI-NEXT: v_cvt_f32_f16_e32 v18, s28
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s78
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT: v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT: v_add_f32_e32 v21, 0x38000000, v21
-; SI-NEXT: v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT: v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT: v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT: v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT: v_add_f32_e32 v26, 0x38000000, v26
-; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v27
-; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x38000000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v30
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT: v_cvt_f16_f32_e32 v20, v20
-; SI-NEXT: v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT: v_cvt_f16_f32_e32 v22, v22
-; SI-NEXT: v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT: v_cvt_f16_f32_e32 v24, v24
-; SI-NEXT: v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT: v_cvt_f16_f32_e32 v26, v26
-; SI-NEXT: v_cvt_f16_f32_e32 v30, v30
-; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
-; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
-; SI-NEXT: v_cvt_f16_f32_e32 v27, v27
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v27
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v23
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v21
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v19
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v15
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v13
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v32
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_or_b32_e32 v54, v16, v8
+; SI-NEXT: v_or_b32_e32 v51, v17, v10
+; SI-NEXT: v_or_b32_e32 v50, v18, v12
+; SI-NEXT: v_or_b32_e32 v48, v19, v14
+; SI-NEXT: v_lshr_b64 v[30:31], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[22:23], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[20:21], v[10:11], 16
+; SI-NEXT: v_lshr_b64 v[18:19], v[12:13], 16
+; SI-NEXT: v_lshr_b64 v[16:17], v[14:15], 16
; SI-NEXT: s_branch .LBB101_6
; SI-NEXT: .LBB101_5:
-; SI-NEXT: v_mov_b32_e32 v1, s37
-; SI-NEXT: v_mov_b32_e32 v0, s36
-; SI-NEXT: v_mov_b32_e32 v3, s35
-; SI-NEXT: v_mov_b32_e32 v2, s34
-; SI-NEXT: v_mov_b32_e32 v5, s31
-; SI-NEXT: v_mov_b32_e32 v4, s30
-; SI-NEXT: v_mov_b32_e32 v7, vcc_hi
-; SI-NEXT: v_mov_b32_e32 v6, vcc_lo
-; SI-NEXT: v_mov_b32_e32 v9, s95
-; SI-NEXT: v_mov_b32_e32 v8, s94
-; SI-NEXT: v_mov_b32_e32 v11, s93
-; SI-NEXT: v_mov_b32_e32 v10, s92
-; SI-NEXT: v_mov_b32_e32 v13, s91
-; SI-NEXT: v_mov_b32_e32 v12, s90
-; SI-NEXT: v_mov_b32_e32 v15, s89
-; SI-NEXT: v_mov_b32_e32 v14, s88
-; SI-NEXT: v_mov_b32_e32 v17, s79
-; SI-NEXT: v_mov_b32_e32 v16, s78
-; SI-NEXT: v_mov_b32_e32 v19, s77
-; SI-NEXT: v_mov_b32_e32 v18, s76
-; SI-NEXT: v_mov_b32_e32 v21, s75
-; SI-NEXT: v_mov_b32_e32 v20, s74
-; SI-NEXT: v_mov_b32_e32 v23, s73
-; SI-NEXT: v_mov_b32_e32 v22, s72
-; SI-NEXT: v_mov_b32_e32 v25, s62
-; SI-NEXT: v_mov_b32_e32 v24, s61
-; SI-NEXT: v_mov_b32_e32 v27, s58
-; SI-NEXT: v_mov_b32_e32 v26, s57
-; SI-NEXT: v_mov_b32_e32 v29, s47
-; SI-NEXT: v_mov_b32_e32 v28, s46
-; SI-NEXT: v_mov_b32_e32 v31, s45
-; SI-NEXT: v_mov_b32_e32 v30, s44
-; SI-NEXT: .LBB101_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v1
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[30:31], 16
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[28:29], 16
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[26:27], 16
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v21, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[24:25], 16
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_mul_f32_e32 v20, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v5
-; SI-NEXT: v_lshr_b64 v[4:5], v[22:23], 16
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_mul_f32_e32 v18, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[20:21], 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[18:19], 16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[16:17], 16
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v9
-; SI-NEXT: v_lshr_b64 v[8:9], v[14:15], 16
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v11
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[10:11], v[32:33], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[38:39], 16
-; SI-NEXT: v_lshr_b64 v[14:15], v[48:49], 16
-; SI-NEXT: v_readlane_b32 s30, v40, 4
-; SI-NEXT: v_lshr_b64 v[15:16], v[50:51], 16
-; SI-NEXT: v_readlane_b32 s31, v40, 5
-; SI-NEXT: v_readlane_b32 s37, v40, 3
+; SI-NEXT: v_mov_b32_e32 v32, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v38, s93
+; SI-NEXT: v_mov_b32_e32 v33, s94
+; SI-NEXT: v_mov_b32_e32 v34, s92
+; SI-NEXT: v_mov_b32_e32 v35, s91
+; SI-NEXT: v_mov_b32_e32 v36, s90
+; SI-NEXT: v_mov_b32_e32 v37, s89
+; SI-NEXT: v_mov_b32_e32 v39, s88
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v5, s11
+; SI-NEXT: v_mov_b32_e32 v7, s13
+; SI-NEXT: v_mov_b32_e32 v9, s41
+; SI-NEXT: v_mov_b32_e32 v11, s47
+; SI-NEXT: v_mov_b32_e32 v48, s59
+; SI-NEXT: v_mov_b32_e32 v50, s45
+; SI-NEXT: v_mov_b32_e32 v51, s40
+; SI-NEXT: v_mov_b32_e32 v54, s36
+; SI-NEXT: v_mov_b32_e32 v49, s35
+; SI-NEXT: v_mov_b32_e32 v52, s31
+; SI-NEXT: v_mov_b32_e32 v53, s30
+; SI-NEXT: v_mov_b32_e32 v55, s4
+; SI-NEXT: v_mov_b32_e32 v13, s43
+; SI-NEXT: v_mov_b32_e32 v15, s57
+; SI-NEXT: v_mov_b32_e32 v30, s8
+; SI-NEXT: v_mov_b32_e32 v28, s14
+; SI-NEXT: v_mov_b32_e32 v26, s44
+; SI-NEXT: v_mov_b32_e32 v24, s58
+; SI-NEXT: v_mov_b32_e32 v22, s60
+; SI-NEXT: v_mov_b32_e32 v20, s62
+; SI-NEXT: v_mov_b32_e32 v18, s72
+; SI-NEXT: v_mov_b32_e32 v16, s74
+; SI-NEXT: .LBB101_6: ; %end
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v55
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v39
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v28
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v53
+; SI-NEXT: v_or_b32_e32 v2, v4, v2
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v37
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v26
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v52
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v36
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v24
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v49
+; SI-NEXT: v_or_b32_e32 v6, v8, v6
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v35
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v54
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v22
+; SI-NEXT: v_or_b32_e32 v8, v8, v10
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v34
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v51
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v20
+; SI-NEXT: v_or_b32_e32 v10, v10, v12
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v33
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v50
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; SI-NEXT: v_or_b32_e32 v12, v12, v14
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v38
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v48
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v32
+; SI-NEXT: v_readlane_b32 s30, v40, 3
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: v_readlane_b32 s31, v40, 4
; SI-NEXT: v_readlane_b32 s36, v40, 2
; SI-NEXT: v_readlane_b32 s35, v40, 1
; SI-NEXT: v_readlane_b32 s34, v40, 0
@@ -73611,304 +72591,240 @@ define <32 x half> @bitcast_v32bf16_to_v32f16(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v37, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v28
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v19
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v17
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v18
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v32
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v63, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v62, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v61, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v59, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v58, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v14
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v47, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v23
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr39
; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB102_2
+; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v39, v1, v40
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v38, v1, v41
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_or_b32_e32 v16, v0, v63
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v37, v1, v42
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_or_b32_e32 v23, v0, v62
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v36, v1, v43
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_or_b32_e32 v24, v0, v61
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v35, v1, v44
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: v_or_b32_e32 v25, v0, v60
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v33, v1, v45
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v27, v0, v58
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v30, v1, v47
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v29, v0, v57
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v28, v1, v59
+; SI-NEXT: v_or_b32_e32 v31, v0, v56
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v14
+; SI-NEXT: v_alignbit_b32 v48, v39, v63, 16
+; SI-NEXT: v_alignbit_b32 v49, v38, v62, 16
+; SI-NEXT: v_alignbit_b32 v50, v37, v61, 16
+; SI-NEXT: v_alignbit_b32 v51, v36, v60, 16
+; SI-NEXT: v_alignbit_b32 v52, v35, v58, 16
+; SI-NEXT: v_alignbit_b32 v53, v33, v57, 16
+; SI-NEXT: v_alignbit_b32 v54, v30, v56, 16
+; SI-NEXT: v_alignbit_b32 v55, v28, v46, 16
+; SI-NEXT: v_or_b32_e32 v34, v0, v46
+; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB102_2
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v31
-; SI-NEXT: v_alignbit_b32 v23, v1, v33, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v47
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v34
-; SI-NEXT: v_alignbit_b32 v24, v23, v0, 16
-; SI-NEXT: v_alignbit_b32 v22, v3, v35, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v46
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v36
-; SI-NEXT: v_alignbit_b32 v25, v22, v0, 16
-; SI-NEXT: v_alignbit_b32 v21, v5, v37, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v44
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v38
-; SI-NEXT: v_alignbit_b32 v26, v21, v0, 16
-; SI-NEXT: v_alignbit_b32 v20, v7, v39, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v42
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v48
-; SI-NEXT: v_alignbit_b32 v27, v20, v0, 16
-; SI-NEXT: v_alignbit_b32 v19, v9, v49, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v41
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v50
-; SI-NEXT: v_alignbit_b32 v28, v19, v0, 16
-; SI-NEXT: v_alignbit_b32 v17, v11, v51, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v40
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v52
-; SI-NEXT: v_alignbit_b32 v29, v17, v0, 16
-; SI-NEXT: v_alignbit_b32 v14, v13, v54, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v55
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v43
-; SI-NEXT: v_alignbit_b32 v30, v14, v0, 16
-; SI-NEXT: v_alignbit_b32 v16, v15, v45, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v53
-; SI-NEXT: v_alignbit_b32 v32, v16, v0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v46
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v44
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v42
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v40
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v55
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v53
-; SI-NEXT: v_alignbit_b32 v0, v0, v63, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v62, 16
-; SI-NEXT: v_alignbit_b32 v4, v4, v61, 16
-; SI-NEXT: v_alignbit_b32 v6, v6, v60, 16
-; SI-NEXT: v_alignbit_b32 v8, v8, v59, 16
-; SI-NEXT: v_alignbit_b32 v10, v10, v58, 16
-; SI-NEXT: v_alignbit_b32 v12, v12, v57, 16
-; SI-NEXT: v_alignbit_b32 v18, v18, v56, 16
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr43
; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr44
; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr46
; SI-NEXT: .LBB102_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v47
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v63
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v63
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v46
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v23
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v44
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v61
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v42
-; SI-NEXT: v_alignbit_b32 v4, v4, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v41
-; SI-NEXT: v_alignbit_b32 v6, v6, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v59
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v40
-; SI-NEXT: v_alignbit_b32 v8, v8, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v55
-; SI-NEXT: v_alignbit_b32 v10, v10, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v53
-; SI-NEXT: v_alignbit_b32 v12, v12, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_alignbit_b32 v18, v14, v1, 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v43
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v52
-; SI-NEXT: v_alignbit_b32 v16, v15, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v54
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v48
+; SI-NEXT: v_alignbit_b32 v16, v16, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v62
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v2
+; SI-NEXT: v_alignbit_b32 v23, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v61
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v4
+; SI-NEXT: v_alignbit_b32 v24, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v60
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v6
+; SI-NEXT: v_alignbit_b32 v25, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v58
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v8
+; SI-NEXT: v_alignbit_b32 v27, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v57
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v10
+; SI-NEXT: v_alignbit_b32 v29, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v56
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v12
+; SI-NEXT: v_alignbit_b32 v31, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v46
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v14
+; SI-NEXT: v_alignbit_b32 v34, v17, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v59
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_alignbit_b32 v28, v17, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v47
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v50
-; SI-NEXT: v_alignbit_b32 v14, v13, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v51
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v14
+; SI-NEXT: v_alignbit_b32 v30, v18, v13, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v45
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v48
-; SI-NEXT: v_alignbit_b32 v17, v11, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v49
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v13
+; SI-NEXT: v_alignbit_b32 v33, v19, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v44
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v38
-; SI-NEXT: v_alignbit_b32 v19, v9, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v39
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v11
+; SI-NEXT: v_alignbit_b32 v35, v20, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v43
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v36
-; SI-NEXT: v_alignbit_b32 v20, v7, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v37
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v9
+; SI-NEXT: v_alignbit_b32 v36, v21, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v42
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v34
-; SI-NEXT: v_alignbit_b32 v21, v5, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v35
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v7
+; SI-NEXT: v_alignbit_b32 v37, v22, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v41
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v5
+; SI-NEXT: v_alignbit_b32 v38, v26, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v40
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v22, v3, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v23, v1, v23, 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_alignbit_b32 v24, v23, v24, 16
-; SI-NEXT: v_alignbit_b32 v25, v22, v25, 16
-; SI-NEXT: v_alignbit_b32 v26, v21, v26, 16
-; SI-NEXT: v_alignbit_b32 v27, v20, v27, 16
-; SI-NEXT: v_alignbit_b32 v28, v19, v28, 16
-; SI-NEXT: v_alignbit_b32 v29, v17, v29, 16
-; SI-NEXT: v_alignbit_b32 v30, v14, v30, 16
-; SI-NEXT: v_alignbit_b32 v32, v16, v32, 16
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v3
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT: v_alignbit_b32 v39, v32, v1, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v48
+; SI-NEXT: v_alignbit_b32 v48, v39, v1, 16
+; SI-NEXT: v_alignbit_b32 v49, v38, v2, 16
+; SI-NEXT: v_alignbit_b32 v50, v37, v4, 16
+; SI-NEXT: v_alignbit_b32 v51, v36, v6, 16
+; SI-NEXT: v_alignbit_b32 v52, v35, v8, 16
+; SI-NEXT: v_alignbit_b32 v53, v33, v10, 16
+; SI-NEXT: v_alignbit_b32 v54, v30, v12, 16
+; SI-NEXT: v_alignbit_b32 v55, v28, v0, 16
; SI-NEXT: .LBB102_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -73927,54 +72843,54 @@ define <32 x half> @bitcast_v32bf16_to_v32f16(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_or_b32_e32 v11, v17, v11
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v30
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_or_b32_e32 v1, v23, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v25
-; SI-NEXT: v_or_b32_e32 v3, v22, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v26
-; SI-NEXT: v_or_b32_e32 v5, v21, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v27
-; SI-NEXT: v_or_b32_e32 v7, v20, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v28
-; SI-NEXT: v_or_b32_e32 v9, v19, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v29
-; SI-NEXT: v_or_b32_e32 v12, v12, v17
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v18
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v32
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v0, v0, v24
-; SI-NEXT: v_or_b32_e32 v2, v2, v23
-; SI-NEXT: v_or_b32_e32 v4, v4, v22
-; SI-NEXT: v_or_b32_e32 v6, v6, v21
-; SI-NEXT: v_or_b32_e32 v8, v8, v20
-; SI-NEXT: v_or_b32_e32 v10, v10, v19
-; SI-NEXT: v_or_b32_e32 v14, v14, v17
-; SI-NEXT: v_or_b32_e32 v15, v16, v15
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v48
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v39
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v32
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v49
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v38
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v26
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v50
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v37
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v22
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v51
+; SI-NEXT: v_or_b32_e32 v6, v6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v36
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v21
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v52
+; SI-NEXT: v_or_b32_e32 v8, v8, v9
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v35
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v53
+; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v33
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v19
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v54
+; SI-NEXT: v_or_b32_e32 v12, v12, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v34
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v55
+; SI-NEXT: v_or_b32_e32 v14, v14, v15
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -75109,479 +74025,363 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-LABEL: bitcast_v32bf16_to_v32f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v2
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v63, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s42
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b64 exec, s[4:5]
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_writelane_b32 v45, s34, 0
+; SI-NEXT: v_writelane_b32 v45, s35, 1
+; SI-NEXT: v_writelane_b32 v45, s30, 2
+; SI-NEXT: v_writelane_b32 v45, s31, 3
+; SI-NEXT: v_readfirstlane_b32 s94, v1
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v0
+; SI-NEXT: s_lshr_b32 s92, s29, 16
+; SI-NEXT: s_lshr_b32 s95, s28, 16
+; SI-NEXT: s_lshr_b32 s91, s27, 16
+; SI-NEXT: s_lshr_b32 s14, s26, 16
+; SI-NEXT: s_lshr_b32 s90, s25, 16
+; SI-NEXT: s_lshr_b32 s12, s24, 16
+; SI-NEXT: s_lshr_b32 s89, s23, 16
+; SI-NEXT: s_lshr_b32 s10, s22, 16
+; SI-NEXT: s_lshr_b32 s88, s21, 16
+; SI-NEXT: s_lshr_b32 s8, s20, 16
+; SI-NEXT: s_lshr_b32 s79, s19, 16
+; SI-NEXT: s_lshr_b32 s6, s18, 16
+; SI-NEXT: s_lshr_b32 s78, s17, 16
+; SI-NEXT: s_lshr_b32 s30, s16, 16
+; SI-NEXT: s_lshr_b32 s93, s94, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v17
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v13
-; SI-NEXT: v_lshr_b64 v[4:5], v[34:35], 16
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v43
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v14
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v31
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v54
-; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v20
-; SI-NEXT: v_lshr_b64 v[21:22], v[36:37], 16
-; SI-NEXT: v_mov_b32_e32 v46, v20
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v53
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v31
-; SI-NEXT: v_lshr_b64 v[19:20], v[20:21], 16
-; SI-NEXT: v_mov_b32_e32 v20, v46
-; SI-NEXT: v_lshr_b64 v[45:46], v[44:45], 16
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v54
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v50
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v63
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[45:46], v[60:61], 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v63
-; SI-NEXT: v_mov_b32_e32 v18, v63
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v55
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v55
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshr_b64 v[22:23], v[23:24], 16
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mov_b32_e32 v29, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v16
-; SI-NEXT: v_lshr_b64 v[1:2], v[8:9], 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v9, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v30
-; SI-NEXT: v_mov_b32_e32 v2, v16
-; SI-NEXT: v_lshr_b64 v[15:16], v[51:52], 16
-; SI-NEXT: v_mov_b32_e32 v16, v2
-; SI-NEXT: v_lshr_b64 v[2:3], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v0, v13
-; SI-NEXT: v_lshr_b64 v[12:13], v[10:11], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[14:15], 16
-; SI-NEXT: v_mov_b32_e32 v14, v29
-; SI-NEXT: v_mov_b32_e32 v29, v2
-; SI-NEXT: v_lshr_b64 v[2:3], v[56:57], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[32:33], 16
-; SI-NEXT: v_lshr_b64 v[7:8], v[38:39], 16
-; SI-NEXT: v_mov_b32_e32 v8, v43
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v30
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v10, v9
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v43, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v16
-; SI-NEXT: v_lshr_b64 v[5:6], v[6:7], 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[40:41], 16
-; SI-NEXT: v_mov_b32_e32 v61, v13
-; SI-NEXT: v_mov_b32_e32 v13, v0
-; SI-NEXT: v_mov_b32_e32 v0, v42
-; SI-NEXT: v_lshr_b64 v[41:42], v[62:63], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v45, v5
-; SI-NEXT: v_lshr_b64 v[5:6], v[58:59], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v63, v18
-; SI-NEXT: v_mov_b32_e32 v42, v0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: s_or_b32 s7, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_mov_b32 s9, s6
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s43, s5, s6
+; SI-NEXT: s_and_b32 s5, s20, 0xffff
+; SI-NEXT: s_mov_b32 s11, s8
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s31, s5, s8
+; SI-NEXT: s_and_b32 s5, s22, 0xffff
+; SI-NEXT: s_mov_b32 s13, s10
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s34, s5, s10
+; SI-NEXT: s_and_b32 s5, s24, 0xffff
+; SI-NEXT: s_mov_b32 s15, s12
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s35, s5, s12
+; SI-NEXT: s_and_b32 s5, s26, 0xffff
+; SI-NEXT: s_mov_b32 s72, s14
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_or_b32 s74, s5, s14
+; SI-NEXT: s_and_b32 s5, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s42, s95, 16
+; SI-NEXT: s_or_b32 s75, s5, s42
+; SI-NEXT: s_and_b32 s5, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s44, vcc_lo, 16
+; SI-NEXT: s_or_b32 s76, s5, s44
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s78, 16
+; SI-NEXT: s_or_b32 s5, s5, s40
+; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
+; SI-NEXT: s_mov_b32 s41, s43
+; SI-NEXT: s_mov_b32 s4, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s79, 16
+; SI-NEXT: s_or_b32 s7, s7, s43
+; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
+; SI-NEXT: s_mov_b32 s6, s9
+; SI-NEXT: s_and_b32 s9, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s88, 16
+; SI-NEXT: s_or_b32 s9, s9, s43
+; SI-NEXT: s_lshr_b64 s[56:57], s[8:9], 16
+; SI-NEXT: s_mov_b32 s8, s11
+; SI-NEXT: s_and_b32 s11, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s89, 16
+; SI-NEXT: s_or_b32 s11, s11, s43
+; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
+; SI-NEXT: s_mov_b32 s10, s13
+; SI-NEXT: s_and_b32 s13, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s90, 16
+; SI-NEXT: s_or_b32 s13, s13, s43
+; SI-NEXT: s_lshr_b64 s[60:61], s[12:13], 16
+; SI-NEXT: s_mov_b32 s12, s15
+; SI-NEXT: s_and_b32 s15, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s91, 16
+; SI-NEXT: s_or_b32 s15, s15, s43
+; SI-NEXT: s_and_b32 s43, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s45, s92, 16
+; SI-NEXT: s_or_b32 s43, s43, s45
+; SI-NEXT: s_and_b32 s45, s94, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s93, 16
+; SI-NEXT: s_or_b32 s45, s45, s47
+; SI-NEXT: s_lshr_b64 s[62:63], s[14:15], 16
+; SI-NEXT: s_mov_b32 s14, s72
+; SI-NEXT: s_lshr_b64 s[72:73], s[42:43], 16
+; SI-NEXT: s_mov_b32 s57, s76
+; SI-NEXT: s_mov_b32 s47, s75
+; SI-NEXT: s_mov_b32 s42, s74
+; SI-NEXT: s_lshr_b64 s[74:75], s[44:45], 16
+; SI-NEXT: s_mov_b64 s[76:77], 0
; SI-NEXT: s_branch .LBB103_3
; SI-NEXT: .LBB103_2:
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: s_mov_b64 s[76:77], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr45
; SI-NEXT: .LBB103_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s44, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s44, 1
; SI-NEXT: s_cbranch_scc1 .LBB103_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v60
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v54
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_lshr_b64 v[44:45], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v56
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v53
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v42
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v7
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v63
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v40
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v18, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v0, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s30, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v0
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
-; SI-NEXT: v_lshr_b64 v[41:42], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v51, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; SI-NEXT: v_lshr_b64 v[15:16], v[51:52], 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[61:62], v[14:15], 16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_lshr_b64 v[19:20], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_lshr_b64 v[20:21], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v7
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_lshr_b64 v[22:23], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_add_f32_e32 v11, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_lshr_b64 v[25:26], v[1:2], 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v0
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; SI-NEXT: v_mov_b32_e32 v1, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[28:29], v[11:12], 16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_mov_b32_e32 v1, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshr_b64 v[11:12], v[8:9], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
-; SI-NEXT: v_lshr_b64 v[7:8], v[38:39], 16
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[21:22], v[36:37], 16
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[45:46], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[23:24], 16
-; SI-NEXT: v_lshr_b64 v[19:20], v[20:21], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_lshr_b64 v[1:2], v[8:9], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v2
-; SI-NEXT: v_lshr_b64 v[4:5], v[34:35], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[10:11], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v5, v44
-; SI-NEXT: .LBB103_5: ; %end
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[11:12], 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v0
+; SI-NEXT: s_lshl_b32 s4, s95, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v0
+; SI-NEXT: s_lshl_b32 s7, s93, 16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; SI-NEXT: s_lshl_b32 s6, s94, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s92, 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[11:12], 16
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v4
+; SI-NEXT: v_add_f32_e32 v23, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s29, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s91, 16
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_add_f32_e32 v28, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s27, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s90, 16
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v4
+; SI-NEXT: v_add_f32_e32 v30, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s25, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s89, 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; SI-NEXT: v_add_f32_e32 v32, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s23, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s88, 16
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v4
+; SI-NEXT: v_add_f32_e32 v34, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s21, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s79, 16
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v4
+; SI-NEXT: v_add_f32_e32 v36, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s19, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: s_lshl_b32 s7, s78, 16
+; SI-NEXT: s_lshl_b32 s4, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v4
+; SI-NEXT: v_add_f32_e32 v38, s6, v0
+; SI-NEXT: s_lshl_b32 s6, s17, 16
+; SI-NEXT: v_add_f32_e32 v4, s7, v0
+; SI-NEXT: v_lshr_b64 v[15:16], v[23:24], 16
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v4
+; SI-NEXT: v_add_f32_e32 v50, s6, v0
+; SI-NEXT: v_add_f32_e32 v4, s5, v0
+; SI-NEXT: v_add_f32_e32 v11, s4, v0
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; SI-NEXT: v_lshr_b64 v[1:2], v[30:31], 16
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; SI-NEXT: v_lshr_b64 v[3:4], v[32:33], 16
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[34:35], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[11:12], 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; SI-NEXT: v_lshr_b64 v[7:8], v[36:37], 16
+; SI-NEXT: v_lshr_b64 v[12:13], v[50:51], 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v10
+; SI-NEXT: v_lshr_b64 v[9:10], v[38:39], 16
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshr_b64 v[40:41], v[11:12], 16
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[41:42], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[17:18], v[28:29], 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshr_b64 v[42:43], v[6:7], 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[43:44], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[10:11], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[52:53], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[14:15], 16
+; SI-NEXT: v_mov_b32_e32 v0, v40
+; SI-NEXT: v_mov_b32_e32 v2, v41
+; SI-NEXT: v_mov_b32_e32 v4, v42
+; SI-NEXT: v_mov_b32_e32 v6, v43
+; SI-NEXT: s_branch .LBB103_6
+; SI-NEXT: .LBB103_5:
+; SI-NEXT: v_mov_b32_e32 v51, s78
+; SI-NEXT: v_mov_b32_e32 v39, s79
+; SI-NEXT: v_mov_b32_e32 v37, s88
+; SI-NEXT: v_mov_b32_e32 v35, s89
+; SI-NEXT: v_mov_b32_e32 v33, s90
+; SI-NEXT: v_mov_b32_e32 v31, s91
+; SI-NEXT: v_mov_b32_e32 v29, s92
+; SI-NEXT: v_mov_b32_e32 v24, s93
+; SI-NEXT: v_mov_b32_e32 v48, s57
+; SI-NEXT: v_mov_b32_e32 v27, s47
+; SI-NEXT: v_mov_b32_e32 v26, s42
+; SI-NEXT: v_mov_b32_e32 v25, s35
+; SI-NEXT: v_mov_b32_e32 v22, s34
+; SI-NEXT: v_mov_b32_e32 v21, s31
+; SI-NEXT: v_mov_b32_e32 v20, s41
+; SI-NEXT: v_mov_b32_e32 v19, s4
+; SI-NEXT: v_mov_b32_e32 v0, s40
+; SI-NEXT: v_mov_b32_e32 v2, s46
+; SI-NEXT: v_mov_b32_e32 v4, s56
+; SI-NEXT: v_mov_b32_e32 v6, s58
+; SI-NEXT: v_mov_b32_e32 v54, s60
+; SI-NEXT: v_mov_b32_e32 v10, s62
+; SI-NEXT: v_mov_b32_e32 v52, s72
+; SI-NEXT: v_mov_b32_e32 v49, s74
+; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v9, s7
+; SI-NEXT: v_mov_b32_e32 v7, s9
+; SI-NEXT: v_mov_b32_e32 v5, s11
+; SI-NEXT: v_mov_b32_e32 v3, s13
+; SI-NEXT: v_mov_b32_e32 v1, s15
+; SI-NEXT: v_mov_b32_e32 v17, s43
+; SI-NEXT: v_mov_b32_e32 v15, s45
+; SI-NEXT: .LBB103_6: ; %end
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v49
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v37
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v61
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v35
-; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v25
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v9
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v22
-; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v24
-; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v45
-; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v39
-; SI-NEXT: v_or_b32_e32 v7, v7, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v19
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v4, v5, v4
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v37
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v5, v7, v5
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v51
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v39
+; SI-NEXT: v_or_b32_e32 v6, v7, v6
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v35
+; SI-NEXT: v_or_b32_e32 v1, v8, v1
+; SI-NEXT: v_or_b32_e32 v3, v9, v3
+; SI-NEXT: v_or_b32_e32 v7, v11, v7
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v54
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v26
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v21
-; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v13
-; SI-NEXT: v_or_b32_e32 v10, v10, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v13
-; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v26
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v33
+; SI-NEXT: v_or_b32_e32 v10, v11, v10
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v31
+; SI-NEXT: v_or_b32_e32 v9, v12, v9
+; SI-NEXT: v_or_b32_e32 v11, v13, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v52
; SI-NEXT: v_or_b32_e32 v12, v12, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v28
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v29
; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v41
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v48
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v49
; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v52
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
+; SI-NEXT: v_readlane_b32 s30, v45, 2
; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: v_readlane_b32 s31, v45, 3
+; SI-NEXT: v_readlane_b32 s35, v45, 1
+; SI-NEXT: v_readlane_b32 s34, v45, 0
+; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -85127,791 +83927,737 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v29, 1.0, v1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v36, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v17
-; SI-NEXT: v_mul_f32_e32 v28, 1.0, v27
-; SI-NEXT: v_mul_f32_e32 v23, 1.0, v23
-; SI-NEXT: v_mul_f32_e32 v27, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v33, 1.0, v32
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
-; SI-NEXT: v_mul_f32_e32 v32, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v39, 1.0, v35
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; SI-NEXT: v_mul_f32_e32 v35, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v37
-; SI-NEXT: v_mul_f32_e32 v51, 1.0, v7
-; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
-; SI-NEXT: v_mul_f32_e32 v49, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v54, 1.0, v36
-; SI-NEXT: v_mul_f32_e32 v55, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
-; SI-NEXT: v_mul_f32_e32 v53, 1.0, v10
-; SI-NEXT: v_mul_f32_e32 v42, 1.0, v30
-; SI-NEXT: v_mul_f32_e32 v43, 1.0, v11
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v22
-; SI-NEXT: v_mul_f32_e32 v41, 1.0, v12
-; SI-NEXT: v_mul_f32_e32 v45, 1.0, v21
-; SI-NEXT: v_mul_f32_e32 v46, 1.0, v13
-; SI-NEXT: v_mul_f32_e32 v52, 1.0, v20
-; SI-NEXT: v_mul_f32_e32 v44, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v56, 1.0, v19
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_mul_f32_e32 v57, 1.0, v15
-; SI-NEXT: v_mul_f32_e32 v40, 1.0, v18
-; SI-NEXT: v_mul_f32_e32 v47, 1.0, v16
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v15
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v23
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v10
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v36
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v29
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v63, 16, v30
+; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v24
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v28
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v27
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v21
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v26
+; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v20
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v18
+; SI-NEXT: ; kill: killed $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; kill: killed $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr57
; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr43
; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; kill: killed $vgpr17
; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr60
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; kill: killed $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; kill: killed $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB108_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v23
-; SI-NEXT: v_alignbit_b32 v37, v1, v29, 16
-; SI-NEXT: v_alignbit_b32 v38, v30, v27, 16
-; SI-NEXT: v_alignbit_b32 v1, v38, v37, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v45, v1, v59
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; SI-NEXT: v_or_b32_e32 v46, v1, v58
+; SI-NEXT: v_alignbit_b32 v1, v46, v45, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v38, v37, 16
+; SI-NEXT: v_alignbit_b32 v1, v46, v45, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v38, v37, 8
+; SI-NEXT: v_alignbit_b32 v1, v46, v45, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v33
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v24
-; SI-NEXT: v_alignbit_b32 v20, v1, v34, 16
-; SI-NEXT: v_alignbit_b32 v21, v19, v32, 16
-; SI-NEXT: v_alignbit_b32 v1, v21, v20, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_or_b32_e32 v38, v1, v61
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v37, v1, v60
+; SI-NEXT: v_alignbit_b32 v1, v37, v38, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v21, v20, 16
+; SI-NEXT: v_alignbit_b32 v1, v37, v38, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v21, v20, 8
+; SI-NEXT: v_alignbit_b32 v1, v37, v38, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v39
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v25
-; SI-NEXT: v_alignbit_b32 v17, v1, v48, 16
-; SI-NEXT: v_alignbit_b32 v18, v16, v35, 16
-; SI-NEXT: v_alignbit_b32 v1, v18, v17, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT: v_or_b32_e32 v34, v1, v63
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v35, v1, v62
+; SI-NEXT: v_alignbit_b32 v1, v35, v34, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v18, v17, 16
+; SI-NEXT: v_alignbit_b32 v1, v35, v34, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v18, v17, 8
+; SI-NEXT: v_alignbit_b32 v1, v35, v34, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v26
-; SI-NEXT: v_alignbit_b32 v14, v1, v51, 16
-; SI-NEXT: v_alignbit_b32 v15, v13, v49, 16
-; SI-NEXT: v_alignbit_b32 v1, v15, v14, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_or_b32_e32 v32, v1, v48
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v33, v1, v39
+; SI-NEXT: v_alignbit_b32 v1, v33, v32, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v15, v14, 16
+; SI-NEXT: v_alignbit_b32 v1, v33, v32, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v15, v14, 8
+; SI-NEXT: v_alignbit_b32 v1, v33, v32, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v54
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v31
-; SI-NEXT: v_alignbit_b32 v11, v1, v55, 16
-; SI-NEXT: v_alignbit_b32 v12, v10, v53, 16
-; SI-NEXT: v_alignbit_b32 v1, v12, v11, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT: v_or_b32_e32 v30, v1, v50
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v31, v1, v49
+; SI-NEXT: v_alignbit_b32 v1, v31, v30, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v12, v11, 16
+; SI-NEXT: v_alignbit_b32 v1, v31, v30, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v12, v11, 8
+; SI-NEXT: v_alignbit_b32 v1, v31, v30, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v42
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v36
-; SI-NEXT: v_alignbit_b32 v8, v1, v43, 16
-; SI-NEXT: v_alignbit_b32 v9, v7, v41, 16
-; SI-NEXT: v_alignbit_b32 v1, v9, v8, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT: v_or_b32_e32 v27, v1, v52
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v28, v1, v51
+; SI-NEXT: v_alignbit_b32 v1, v28, v27, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v9, v8, 16
+; SI-NEXT: v_alignbit_b32 v1, v28, v27, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v9, v8, 8
+; SI-NEXT: v_alignbit_b32 v1, v28, v27, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v45
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v52
-; SI-NEXT: v_alignbit_b32 v5, v1, v46, 16
-; SI-NEXT: v_alignbit_b32 v6, v4, v44, 16
-; SI-NEXT: v_alignbit_b32 v1, v6, v5, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: v_or_b32_e32 v25, v1, v54
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v26, v1, v53
+; SI-NEXT: v_alignbit_b32 v1, v26, v25, 24
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v6, v5, 16
+; SI-NEXT: v_alignbit_b32 v1, v26, v25, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v1, v6, v5, 8
+; SI-NEXT: v_alignbit_b32 v1, v26, v25, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v56
-; SI-NEXT: v_alignbit_b32 v2, v1, v57, 16
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v40
-; SI-NEXT: v_alignbit_b32 v3, v1, v47, 16
-; SI-NEXT: v_alignbit_b32 v22, v3, v2, 24
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v17, v1, v40
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v16
+; SI-NEXT: v_or_b32_e32 v23, v1, v55
+; SI-NEXT: v_alignbit_b32 v1, v23, v17, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v3, v2, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v23, v17, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v3, v2, 8
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v23, v17, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 24, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v21
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v35
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v18
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v33
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v26
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v31
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v15
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v28
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v31
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v26
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v36
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v21, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v40
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v20, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v6
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_bfe_u32 v1, v19, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v3
-; SI-NEXT: v_lshrrev_b32_e32 v62, 8, v38
-; SI-NEXT: v_lshrrev_b32_e32 v61, 24, v24
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v25
-; SI-NEXT: v_lshrrev_b32_e32 v58, 24, v52
-; SI-NEXT: v_lshrrev_b32_e32 v63, 8, v12
-; SI-NEXT: v_lshrrev_b32_e32 v60, 8, v9
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: v_bfe_u32 v1, v18, 8, 8
+; SI-NEXT: v_lshrrev_b32_e32 v47, 8, v46
+; SI-NEXT: v_lshrrev_b32_e32 v42, 8, v37
+; SI-NEXT: v_lshrrev_b32_e32 v41, 8, v23
+; SI-NEXT: v_bfe_u32 v57, v36, 8, 8
+; SI-NEXT: v_bfe_u32 v56, v29, 8, 8
+; SI-NEXT: v_bfe_u32 v44, v24, 8, 8
+; SI-NEXT: v_bfe_u32 v43, v22, 8, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr61
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr62
; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr39
; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr44
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr57
; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: .LBB108_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB108_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v33
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_alignbit_b32 v37, v28, v22, 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v39
-; SI-NEXT: v_alignbit_b32 v20, v20, v19, 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v23
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v19
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v24
-; SI-NEXT: v_alignbit_b32 v38, v30, v22, 16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v40
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v19
-; SI-NEXT: v_alignbit_b32 v22, v38, v37, 24
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v50
-; SI-NEXT: v_alignbit_b32 v17, v17, v16, 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v35
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v24
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v17, v17, v15, 16
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v55
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v16
+; SI-NEXT: v_alignbit_b32 v23, v18, v15, 16
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v54
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT: v_alignbit_b32 v25, v15, v13, 16
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v53
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v14
+; SI-NEXT: v_alignbit_b32 v26, v19, v13, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v52
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_alignbit_b32 v27, v13, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v51
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v12
+; SI-NEXT: v_alignbit_b32 v28, v20, v11, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v50
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_alignbit_b32 v30, v11, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v49
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v10
+; SI-NEXT: v_alignbit_b32 v31, v21, v9, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v48
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_alignbit_b32 v32, v9, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v39
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v8
+; SI-NEXT: v_alignbit_b32 v33, v22, v7, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v63
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_alignbit_b32 v34, v7, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v62
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
+; SI-NEXT: v_alignbit_b32 v35, v24, v5, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v61
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_alignbit_b32 v38, v5, v3, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v60
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v4
+; SI-NEXT: v_alignbit_b32 v37, v29, v3, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v38, v37, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v25
-; SI-NEXT: v_alignbit_b32 v21, v19, v21, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v59
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_alignbit_b32 v45, v3, v1, 16
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v58
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v2
+; SI-NEXT: v_alignbit_b32 v46, v36, v1, 16
+; SI-NEXT: v_alignbit_b32 v1, v46, v45, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v38, v37, 8
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v46, v45, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v21, v20, 24
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v54
-; SI-NEXT: v_alignbit_b32 v14, v14, v13, 16
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v49
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v25
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v46, v45, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v21, v20, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v26
-; SI-NEXT: v_alignbit_b32 v18, v16, v18, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v37, v38, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v21, v20, 8
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v13
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v37, v38, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v18, v17, 24
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v42
-; SI-NEXT: v_alignbit_b32 v11, v11, v10, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v53
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v26
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v37, v38, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v18, v17, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v31
-; SI-NEXT: v_alignbit_b32 v15, v13, v15, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v34, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v18, v17, 8
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v10
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v34, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v15, v14, 24
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v45
-; SI-NEXT: v_alignbit_b32 v8, v8, v7, 16
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v41
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v31
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v35, v34, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v15, v14, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v36
-; SI-NEXT: v_alignbit_b32 v12, v10, v12, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v33, v32, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v15, v14, 8
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v7
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v33, v32, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_alignbit_b32 v1, v33, v32, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v12, v11, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v56
-; SI-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v44
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v36
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v31, v30, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v12, v11, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v52
-; SI-NEXT: v_alignbit_b32 v9, v7, v9, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v31, v30, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v12, v11, 8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v4
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v31, v30, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v9, v8, 24
-; SI-NEXT: v_alignbit_b32 v2, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v47
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v52
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v28, v27, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v9, v8, 16
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v40
-; SI-NEXT: v_alignbit_b32 v6, v4, v6, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v28, v27, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v9, v8, 8
-; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v1
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v28, v27, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v6, v5, 24
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v40
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v26, v25, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v6, v5, 16
-; SI-NEXT: v_alignbit_b32 v3, v1, v3, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v26, v25, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v6, v5, 8
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v26, v25, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v3, v2, 24
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v23, v17, 24
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v3, v2, 16
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v23, v17, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_alignbit_b32 v22, v3, v2, 8
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v23, v17, 8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 8, v21
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v35
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 8, v18
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v33
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 8, v15
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v10
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v22, 24, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v26
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v12
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v31
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v14
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v36
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v40
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v31
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v6
-; SI-NEXT: v_lshrrev_b32_e32 v62, 8, v38
-; SI-NEXT: v_lshrrev_b32_e32 v61, 24, v24
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v25
-; SI-NEXT: v_lshrrev_b32_e32 v58, 24, v52
-; SI-NEXT: v_lshrrev_b32_e32 v63, 8, v12
-; SI-NEXT: v_lshrrev_b32_e32 v60, 8, v9
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v28
+; SI-NEXT: v_lshrrev_b32_e32 v47, 8, v46
+; SI-NEXT: v_lshrrev_b32_e32 v42, 8, v37
+; SI-NEXT: v_lshrrev_b32_e32 v57, 24, v2
+; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v4
+; SI-NEXT: v_lshrrev_b32_e32 v44, 24, v6
+; SI-NEXT: v_lshrrev_b32_e32 v43, 24, v8
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 8, v3
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v26
+; SI-NEXT: v_lshrrev_b32_e32 v41, 8, v23
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: .LBB108_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xff, v37
-; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v22
-; SI-NEXT: v_and_b32_e32 v20, 0xff, v20
-; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v45
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v24
-; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v25
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v24, 0xff, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_or_b32_e32 v24, v25, v24
-; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xff, v38
-; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v62
-; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: v_and_b32_e32 v24, 0xff, v30
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT: v_or_b32_e32 v22, v22, v24
-; SI-NEXT: v_or_b32_e32 v22, v23, v22
-; SI-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:4
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v46
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v47
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v36
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v57
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v38
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 8, v22
-; SI-NEXT: v_or_b32_e32 v20, v20, v22
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v23, 24, v23
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_or_b32_e32 v22, v23, v22
-; SI-NEXT: v_or_b32_e32 v20, v20, v22
-; SI-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v20, 0xff, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v21, 8, v21
-; SI-NEXT: v_or_b32_e32 v20, v20, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v61
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: v_or_b32_e32 v19, v21, v19
-; SI-NEXT: v_or_b32_e32 v19, v20, v19
-; SI-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:12
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v37
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v42
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v56
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v34
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 8, v19
-; SI-NEXT: v_or_b32_e32 v17, v17, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v20, 24, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT: v_or_b32_e32 v19, v20, v19
-; SI-NEXT: v_or_b32_e32 v17, v17, v19
-; SI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xff, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v18, 8, v18
-; SI-NEXT: v_or_b32_e32 v17, v17, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 24, v59
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT: v_or_b32_e32 v16, v18, v16
-; SI-NEXT: v_or_b32_e32 v16, v17, v16
-; SI-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:20
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 8, v16
-; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v17
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v35
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v44
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_or_b32_e32 v16, v17, v16
-; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v32
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 8, v15
-; SI-NEXT: v_or_b32_e32 v14, v14, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v33
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v43
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v15
-; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:28
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v22
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v30
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 8, v13
-; SI-NEXT: v_or_b32_e32 v11, v11, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 24, v14
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: v_or_b32_e32 v11, v11, v13
-; SI-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 8, v63
-; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v31
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v12
-; SI-NEXT: v_or_b32_e32 v10, v12, v10
-; SI-NEXT: v_or_b32_e32 v10, v11, v10
-; SI-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v27
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: v_or_b32_e32 v8, v8, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v11
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v11, v10
-; SI-NEXT: v_or_b32_e32 v8, v8, v10
-; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 8, v60
-; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v28
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v9
-; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_or_b32_e32 v7, v8, v7
-; SI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:44
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v25
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT: v_or_b32_e32 v5, v5, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_or_b32_e32 v7, v8, v7
-; SI-NEXT: v_or_b32_e32 v5, v5, v7
-; SI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v26
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v58
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v4, v5, v4
-; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:52
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v17
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v4, v5, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v23
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v41
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v18
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
@@ -88059,707 +86805,725 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-LABEL: bitcast_v32bf16_to_v64i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v40, s34, 0
-; SI-NEXT: v_writelane_b32 v40, s35, 1
-; SI-NEXT: v_writelane_b32 v40, s36, 2
-; SI-NEXT: v_writelane_b32 v40, s37, 3
-; SI-NEXT: v_writelane_b32 v40, s38, 4
-; SI-NEXT: v_writelane_b32 v40, s39, 5
-; SI-NEXT: v_writelane_b32 v40, s48, 6
-; SI-NEXT: v_writelane_b32 v40, s49, 7
-; SI-NEXT: v_writelane_b32 v40, s50, 8
-; SI-NEXT: v_writelane_b32 v40, s51, 9
-; SI-NEXT: v_writelane_b32 v40, s52, 10
-; SI-NEXT: v_writelane_b32 v40, s53, 11
-; SI-NEXT: v_writelane_b32 v40, s54, 12
-; SI-NEXT: v_writelane_b32 v40, s55, 13
-; SI-NEXT: v_writelane_b32 v40, s64, 14
-; SI-NEXT: v_writelane_b32 v40, s65, 15
-; SI-NEXT: v_writelane_b32 v40, s66, 16
-; SI-NEXT: v_writelane_b32 v40, s67, 17
-; SI-NEXT: v_writelane_b32 v40, s68, 18
-; SI-NEXT: v_writelane_b32 v40, s69, 19
-; SI-NEXT: v_writelane_b32 v40, s70, 20
-; SI-NEXT: v_writelane_b32 v40, s71, 21
-; SI-NEXT: v_writelane_b32 v40, s80, 22
-; SI-NEXT: v_writelane_b32 v40, s81, 23
-; SI-NEXT: v_writelane_b32 v40, s82, 24
-; SI-NEXT: v_writelane_b32 v40, s83, 25
-; SI-NEXT: v_writelane_b32 v40, s84, 26
-; SI-NEXT: v_writelane_b32 v40, s85, 27
-; SI-NEXT: v_writelane_b32 v40, s86, 28
-; SI-NEXT: v_writelane_b32 v40, s87, 29
-; SI-NEXT: v_writelane_b32 v40, s96, 30
-; SI-NEXT: v_writelane_b32 v40, s97, 31
-; SI-NEXT: v_writelane_b32 v40, s98, 32
-; SI-NEXT: v_writelane_b32 v40, s99, 33
-; SI-NEXT: v_writelane_b32 v40, s30, 34
-; SI-NEXT: v_writelane_b32 v40, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s42, v2
-; SI-NEXT: v_readfirstlane_b32 s44, v1
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
-; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s24, 16
-; SI-NEXT: s_and_b32 s24, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s23, s23, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readfirstlane_b32 s46, v3
-; SI-NEXT: s_cmp_lg_u32 s46, 0
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s42
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_writelane_b32 v11, s34, 0
+; SI-NEXT: v_writelane_b32 v11, s35, 1
+; SI-NEXT: v_writelane_b32 v11, s36, 2
+; SI-NEXT: v_writelane_b32 v11, s37, 3
+; SI-NEXT: v_writelane_b32 v11, s38, 4
+; SI-NEXT: v_writelane_b32 v11, s39, 5
+; SI-NEXT: v_writelane_b32 v11, s48, 6
+; SI-NEXT: v_writelane_b32 v11, s49, 7
+; SI-NEXT: v_writelane_b32 v11, s50, 8
+; SI-NEXT: v_writelane_b32 v11, s51, 9
+; SI-NEXT: v_writelane_b32 v11, s52, 10
+; SI-NEXT: v_writelane_b32 v11, s53, 11
+; SI-NEXT: v_writelane_b32 v11, s54, 12
+; SI-NEXT: v_writelane_b32 v11, s55, 13
+; SI-NEXT: v_writelane_b32 v11, s64, 14
+; SI-NEXT: v_writelane_b32 v11, s65, 15
+; SI-NEXT: v_writelane_b32 v11, s66, 16
+; SI-NEXT: v_writelane_b32 v11, s67, 17
+; SI-NEXT: v_writelane_b32 v11, s68, 18
+; SI-NEXT: v_writelane_b32 v11, s69, 19
+; SI-NEXT: v_writelane_b32 v11, s70, 20
+; SI-NEXT: v_writelane_b32 v11, s71, 21
+; SI-NEXT: v_writelane_b32 v11, s80, 22
+; SI-NEXT: v_writelane_b32 v11, s81, 23
+; SI-NEXT: v_writelane_b32 v11, s82, 24
+; SI-NEXT: v_writelane_b32 v11, s83, 25
+; SI-NEXT: v_writelane_b32 v11, s84, 26
+; SI-NEXT: v_writelane_b32 v11, s85, 27
+; SI-NEXT: v_writelane_b32 v11, s86, 28
+; SI-NEXT: v_writelane_b32 v11, s87, 29
+; SI-NEXT: v_writelane_b32 v11, s96, 30
+; SI-NEXT: v_writelane_b32 v11, s97, 31
+; SI-NEXT: v_writelane_b32 v11, s98, 32
+; SI-NEXT: v_writelane_b32 v11, s99, 33
+; SI-NEXT: v_writelane_b32 v11, s30, 34
+; SI-NEXT: v_writelane_b32 v11, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s87, v2
+; SI-NEXT: v_readfirstlane_b32 s97, v1
+; SI-NEXT: s_lshr_b32 s7, s29, 16
+; SI-NEXT: s_lshr_b32 s86, s28, 16
+; SI-NEXT: s_lshr_b32 s9, s27, 16
+; SI-NEXT: s_lshr_b32 s85, s26, 16
+; SI-NEXT: s_lshr_b32 s11, s25, 16
+; SI-NEXT: s_lshr_b32 s84, s24, 16
+; SI-NEXT: s_lshr_b32 s13, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s15, s21, 16
+; SI-NEXT: s_lshr_b32 s82, s20, 16
+; SI-NEXT: s_lshr_b32 s41, s19, 16
+; SI-NEXT: s_lshr_b32 s69, s18, 16
+; SI-NEXT: s_lshr_b32 s43, s17, 16
+; SI-NEXT: s_lshr_b32 s88, s16, 16
+; SI-NEXT: s_lshr_b32 s5, s87, 16
+; SI-NEXT: s_lshr_b32 s96, s97, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v3
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: ; implicit-def: $vgpr12 : SGPR spill to VGPR lane
; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v4
-; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v8
-; SI-NEXT: s_lshr_b64 s[58:59], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: s_lshr_b32 s61, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v11
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v12
-; SI-NEXT: s_lshr_b64 s[44:45], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: s_lshr_b32 s47, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: s_lshr_b64 s[24:25], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v13
-; SI-NEXT: s_lshr_b32 s27, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: s_lshr_b32 s43, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v24
-; SI-NEXT: s_lshr_b64 s[20:21], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: s_lshr_b32 s19, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v30
-; SI-NEXT: s_lshr_b64 s[12:13], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: s_lshr_b32 s11, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v36
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v37
-; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v33
-; SI-NEXT: v_readfirstlane_b32 s72, v2
-; SI-NEXT: v_readfirstlane_b32 s60, v6
-; SI-NEXT: v_readfirstlane_b32 s46, v10
-; SI-NEXT: v_readfirstlane_b32 s26, v14
-; SI-NEXT: v_readfirstlane_b32 s42, v18
-; SI-NEXT: v_readfirstlane_b32 s18, v22
-; SI-NEXT: v_readfirstlane_b32 s10, v27
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v34
-; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[60:61], 16
-; SI-NEXT: s_lshr_b64 s[56:57], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[40:41], s[26:27], 16
-; SI-NEXT: s_lshr_b64 s[22:23], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[28:29], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
-; SI-NEXT: s_mov_b32 s75, s76
-; SI-NEXT: s_mov_b32 s59, s62
-; SI-NEXT: s_mov_b32 s45, s56
-; SI-NEXT: s_mov_b32 s25, s40
-; SI-NEXT: s_mov_b32 s15, s22
-; SI-NEXT: s_mov_b32 s21, s28
-; SI-NEXT: s_mov_b32 s13, s16
-; SI-NEXT: s_mov_b32 s7, s8
-; SI-NEXT: s_lshr_b64 s[88:89], s[74:75], 24
-; SI-NEXT: s_lshr_b64 s[90:91], s[74:75], 16
-; SI-NEXT: s_lshr_b64 s[92:93], s[74:75], 8
-; SI-NEXT: s_lshr_b64 s[94:95], s[58:59], 24
-; SI-NEXT: s_lshr_b64 vcc, s[58:59], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[58:59], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[44:45], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[48:49], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[50:51], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[52:53], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[54:55], s[14:15], 24
-; SI-NEXT: s_lshr_b64 s[64:65], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[66:67], s[14:15], 8
-; SI-NEXT: s_lshr_b64 s[68:69], s[20:21], 24
-; SI-NEXT: s_lshr_b64 s[70:71], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[80:81], s[20:21], 8
-; SI-NEXT: v_lshrrev_b32_e32 v48, 24, v1
-; SI-NEXT: v_lshrrev_b32_e32 v39, 24, v5
-; SI-NEXT: s_lshr_b32 s23, s76, 8
-; SI-NEXT: v_lshrrev_b32_e32 v38, 24, v9
-; SI-NEXT: s_lshr_b32 s21, s62, 8
-; SI-NEXT: v_lshrrev_b32_e32 v35, 24, v13
-; SI-NEXT: s_lshr_b32 s18, s56, 8
-; SI-NEXT: v_lshrrev_b32_e32 v32, 24, v17
-; SI-NEXT: s_lshr_b32 s17, s40, 8
-; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v21
-; SI-NEXT: s_lshr_b32 s15, s22, 8
-; SI-NEXT: v_lshrrev_b32_e32 v28, 24, v26
-; SI-NEXT: s_lshr_b32 s10, s28, 8
-; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v33
-; SI-NEXT: s_lshr_b32 s26, s16, 8
-; SI-NEXT: s_mov_b32 s42, s8
-; SI-NEXT: s_lshr_b32 s4, s8, 8
-; SI-NEXT: s_lshr_b64 s[96:97], s[12:13], 24
-; SI-NEXT: s_lshr_b64 s[98:99], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[12:13], 8
-; SI-NEXT: s_lshr_b64 s[82:83], s[6:7], 24
-; SI-NEXT: s_lshr_b64 s[84:85], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[86:87], s[6:7], 8
-; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s88, 16
+; SI-NEXT: s_or_b32 s76, s4, s6
+; SI-NEXT: s_and_b32 s4, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s43, 16
+; SI-NEXT: s_or_b32 s77, s4, s6
+; SI-NEXT: s_lshr_b64 s[44:45], s[76:77], 24
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v12, s44, 2
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s69, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 3
+; SI-NEXT: s_lshr_b64 s[44:45], s[76:77], 16
+; SI-NEXT: s_or_b32 s72, s4, s6
+; SI-NEXT: s_and_b32 s4, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s41, 16
+; SI-NEXT: v_writelane_b32 v12, s44, 0
+; SI-NEXT: s_or_b32 s73, s4, s6
+; SI-NEXT: v_writelane_b32 v12, s45, 1
+; SI-NEXT: s_lshr_b64 s[44:45], s[72:73], 24
+; SI-NEXT: v_writelane_b32 v12, s44, 10
+; SI-NEXT: v_writelane_b32 v12, s45, 11
+; SI-NEXT: s_lshr_b64 s[44:45], s[72:73], 16
+; SI-NEXT: v_writelane_b32 v12, s44, 8
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s82, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 9
+; SI-NEXT: s_lshr_b64 s[44:45], s[72:73], 8
+; SI-NEXT: s_or_b32 s60, s4, s6
+; SI-NEXT: s_and_b32 s4, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s15, 16
+; SI-NEXT: v_writelane_b32 v12, s44, 6
+; SI-NEXT: s_or_b32 s61, s4, s6
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s83, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 7
+; SI-NEXT: s_lshr_b64 s[44:45], s[60:61], 24
+; SI-NEXT: s_or_b32 s62, s4, s6
+; SI-NEXT: s_and_b32 s4, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s13, 16
+; SI-NEXT: v_writelane_b32 v12, s44, 16
+; SI-NEXT: s_or_b32 s63, s4, s6
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s84, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 17
+; SI-NEXT: s_lshr_b64 s[44:45], s[60:61], 16
+; SI-NEXT: s_or_b32 s58, s4, s6
+; SI-NEXT: s_and_b32 s4, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s11, 16
+; SI-NEXT: v_writelane_b32 v12, s44, 14
+; SI-NEXT: s_or_b32 s59, s4, s6
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s85, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 15
+; SI-NEXT: s_lshr_b64 s[44:45], s[60:61], 8
+; SI-NEXT: s_or_b32 s56, s4, s6
+; SI-NEXT: s_and_b32 s4, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s9, 16
+; SI-NEXT: v_writelane_b32 v12, s44, 12
+; SI-NEXT: s_or_b32 s57, s4, s6
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s86, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 13
+; SI-NEXT: s_lshr_b64 s[44:45], s[62:63], 24
+; SI-NEXT: s_or_b32 s46, s4, s6
+; SI-NEXT: s_and_b32 s4, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s7, 16
+; SI-NEXT: v_writelane_b32 v12, s44, 18
+; SI-NEXT: s_or_b32 s47, s4, s6
+; SI-NEXT: s_and_b32 s4, s97, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s96, 16
+; SI-NEXT: v_writelane_b32 v12, s45, 19
+; SI-NEXT: s_or_b32 s44, s4, s6
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s5, 16
+; SI-NEXT: s_or_b32 s45, s4, s6
+; SI-NEXT: s_lshr_b32 s4, s61, 8
+; SI-NEXT: v_writelane_b32 v12, s4, 25
+; SI-NEXT: s_lshr_b32 s4, s63, 8
+; SI-NEXT: v_writelane_b32 v12, s4, 24
+; SI-NEXT: s_lshr_b32 s4, s59, 8
+; SI-NEXT: v_writelane_b32 v12, s4, 23
+; SI-NEXT: s_lshr_b32 s4, s57, 8
+; SI-NEXT: v_writelane_b32 v12, s4, 22
+; SI-NEXT: s_lshr_b32 s4, s47, 8
+; SI-NEXT: v_writelane_b32 v12, s4, 21
+; SI-NEXT: s_lshr_b32 s4, s45, 8
+; SI-NEXT: v_writelane_b32 v12, s4, 20
+; SI-NEXT: s_mov_b32 s6, s77
+; SI-NEXT: s_lshr_b64 vcc, s[76:77], 8
+; SI-NEXT: s_bfe_u32 s75, s7, 0x80008
+; SI-NEXT: v_writelane_b32 v12, s6, 4
+; SI-NEXT: s_lshr_b32 s89, s77, 8
+; SI-NEXT: s_lshr_b32 vcc_hi, s73, 8
+; SI-NEXT: s_bfe_u32 s98, s43, 0x80008
+; SI-NEXT: s_bfe_u32 s99, s41, 0x80008
+; SI-NEXT: s_bfe_u32 s4, s15, 0x80008
+; SI-NEXT: s_bfe_u32 s40, s13, 0x80008
+; SI-NEXT: s_bfe_u32 s42, s11, 0x80008
+; SI-NEXT: s_bfe_u32 s68, s9, 0x80008
+; SI-NEXT: s_bfe_u32 s79, s5, 0x80008
+; SI-NEXT: s_lshr_b64 s[94:95], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[62:63], 8
+; SI-NEXT: s_lshr_b64 s[48:49], s[58:59], 24
+; SI-NEXT: s_lshr_b64 s[52:53], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[66:67], s[58:59], 8
+; SI-NEXT: s_lshr_b64 s[70:71], s[56:57], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[56:57], 8
+; SI-NEXT: s_lshr_b64 s[38:39], s[46:47], 24
+; SI-NEXT: s_lshr_b64 s[50:51], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[46:47], 8
+; SI-NEXT: s_lshr_b64 s[92:93], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[90:91], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[44:45], 8
+; SI-NEXT: s_mov_b64 s[64:65], 0
+; SI-NEXT: v_writelane_b32 v12, s7, 5
+; SI-NEXT: s_mov_b32 s6, s73
+; SI-NEXT: s_mov_b32 s8, s61
+; SI-NEXT: s_mov_b32 s10, s63
+; SI-NEXT: s_mov_b32 s12, s59
+; SI-NEXT: s_mov_b32 s14, s57
+; SI-NEXT: s_mov_b32 s78, s47
+; SI-NEXT: s_mov_b32 s74, s45
; SI-NEXT: s_branch .LBB109_3
; SI-NEXT: .LBB109_2:
-; SI-NEXT: s_mov_b64 s[78:79], -1
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: s_mov_b64 s[64:65], -1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v12, s4, 0
+; SI-NEXT: v_writelane_b32 v12, s5, 1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr79
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr17
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: v_writelane_b32 v12, s4, 2
+; SI-NEXT: v_writelane_b32 v12, s5, 3
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: v_writelane_b32 v12, s4, 4
+; SI-NEXT: v_writelane_b32 v12, s5, 5
+; SI-NEXT: v_writelane_b32 v12, s6, 6
+; SI-NEXT: v_writelane_b32 v12, s7, 7
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: v_writelane_b32 v12, s4, 8
+; SI-NEXT: v_writelane_b32 v12, s5, 9
+; SI-NEXT: v_writelane_b32 v12, s6, 10
+; SI-NEXT: v_writelane_b32 v12, s7, 11
+; SI-NEXT: v_writelane_b32 v12, s8, 12
+; SI-NEXT: v_writelane_b32 v12, s9, 13
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v12, s8, 14
+; SI-NEXT: v_writelane_b32 v12, s9, 15
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; kill: killed $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; kill: killed $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; kill: killed $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; kill: killed $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; kill: killed $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: v_writelane_b32 v12, s8, 16
+; SI-NEXT: v_writelane_b32 v12, s9, 17
+; SI-NEXT: v_writelane_b32 v12, s10, 18
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: v_writelane_b32 v12, s11, 19
+; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: .LBB109_3: ; %Flow
-; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s7, 1
-; SI-NEXT: s_mov_b32 s78, s82
-; SI-NEXT: s_mov_b32 s82, s84
-; SI-NEXT: s_mov_b32 s84, s86
-; SI-NEXT: s_mov_b32 s86, s96
-; SI-NEXT: s_mov_b32 s96, s98
+; SI-NEXT: s_and_b64 s[64:65], s[64:65], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
; SI-NEXT: s_cbranch_scc1 .LBB109_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v36
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v37
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_readfirstlane_b32 s4, v28
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v25
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v25
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v25
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: s_lshl_b32 s4, s97, 16
+; SI-NEXT: v_mov_b32_e32 v4, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s96, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_lshr_b32 s45, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s87, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s44, v1
+; SI-NEXT: v_add_f32_e32 v1, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v1
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v28
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v30
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: s_lshr_b32 s11, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s10, v28
-; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: s_lshr_b64 s[12:13], s[10:11], 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
+; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s86, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v3
+; SI-NEXT: s_lshr_b32 s47, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_readfirstlane_b32 s46, v2
+; SI-NEXT: v_add_f32_e32 v2, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: s_lshr_b32 s7, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v3
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s85, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v5
+; SI-NEXT: s_lshr_b32 s29, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_readfirstlane_b32 s28, v3
+; SI-NEXT: v_add_f32_e32 v3, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v3
+; SI-NEXT: s_lshr_b32 s9, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: v_readfirstlane_b32 s8, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s84, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v6
+; SI-NEXT: s_lshr_b64 s[56:57], s[28:29], 16
+; SI-NEXT: s_lshr_b32 s29, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: v_readfirstlane_b32 s28, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v9, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v9
; SI-NEXT: s_lshr_b32 s11, s4, 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: v_readfirstlane_b32 s10, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s83, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v6
+; SI-NEXT: s_lshr_b64 s[58:59], s[28:29], 16
+; SI-NEXT: s_lshr_b32 s29, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: v_readfirstlane_b32 s28, v5
+; SI-NEXT: v_add_f32_e32 v5, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v5
+; SI-NEXT: s_lshr_b32 s13, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_readfirstlane_b32 s12, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s82, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: s_lshr_b64 s[62:63], s[28:29], 16
+; SI-NEXT: s_lshr_b32 s29, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: v_readfirstlane_b32 s28, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v10, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_lshr_b32 s15, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_readfirstlane_b32 s14, v24
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_readfirstlane_b32 s14, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s69, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: s_lshr_b64 s[60:61], s[28:29], 16
+; SI-NEXT: s_lshr_b32 s29, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: v_readfirstlane_b32 s28, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s41, 16
+; SI-NEXT: v_add_f32_e32 v7, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: s_lshr_b32 s41, s4, 16
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_readfirstlane_b32 s40, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s88, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_lshr_b32 s19, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT: s_lshr_b64 s[20:21], s[14:15], 16
-; SI-NEXT: s_lshr_b32 s15, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_readfirstlane_b32 s18, v6
+; SI-NEXT: v_add_f32_e32 v6, s4, v4
+; SI-NEXT: s_lshl_b32 s4, s43, 16
+; SI-NEXT: v_add_f32_e32 v4, s4, v4
+; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_lshr_b32 s43, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: s_lshr_b32 s25, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v13
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: s_lshr_b32 s27, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v11
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: s_lshr_b32 s45, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: s_lshr_b32 s47, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: s_lshr_b32 s59, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: s_lshr_b32 s61, s4, 16
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s72, v4
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: v_readfirstlane_b32 s10, v27
-; SI-NEXT: v_readfirstlane_b32 s18, v22
-; SI-NEXT: v_readfirstlane_b32 s14, v20
-; SI-NEXT: v_readfirstlane_b32 s42, v18
-; SI-NEXT: v_readfirstlane_b32 s24, v16
-; SI-NEXT: v_readfirstlane_b32 s26, v14
-; SI-NEXT: v_readfirstlane_b32 s44, v12
-; SI-NEXT: v_readfirstlane_b32 s46, v10
-; SI-NEXT: v_readfirstlane_b32 s58, v8
-; SI-NEXT: v_readfirstlane_b32 s60, v6
-; SI-NEXT: s_lshr_b64 s[74:75], s[72:73], 16
-; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s72, v2
-; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[28:29], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[22:23], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[24:25], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[40:41], s[26:27], 16
+; SI-NEXT: v_readfirstlane_b32 s42, v6
+; SI-NEXT: s_lshr_b64 s[20:21], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[42:43], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[56:57], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[60:61], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
-; SI-NEXT: s_mov_b32 s7, s8
-; SI-NEXT: s_mov_b32 s13, s16
-; SI-NEXT: s_mov_b32 s21, s28
-; SI-NEXT: s_mov_b32 s15, s22
-; SI-NEXT: s_mov_b32 s25, s40
-; SI-NEXT: s_mov_b32 s45, s56
-; SI-NEXT: s_mov_b32 s59, s62
-; SI-NEXT: s_mov_b32 s75, s76
-; SI-NEXT: s_lshr_b64 s[88:89], s[74:75], 24
-; SI-NEXT: s_lshr_b64 s[90:91], s[74:75], 16
-; SI-NEXT: s_lshr_b64 s[92:93], s[74:75], 8
-; SI-NEXT: s_lshr_b64 s[94:95], s[58:59], 24
-; SI-NEXT: s_lshr_b64 vcc, s[58:59], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[58:59], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[44:45], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[48:49], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[50:51], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[52:53], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[54:55], s[14:15], 24
-; SI-NEXT: s_lshr_b64 s[64:65], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[66:67], s[14:15], 8
-; SI-NEXT: s_lshr_b64 s[68:69], s[20:21], 24
-; SI-NEXT: s_lshr_b64 s[70:71], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[80:81], s[20:21], 8
-; SI-NEXT: s_lshr_b32 s23, s76, 8
-; SI-NEXT: s_lshr_b32 s21, s62, 8
-; SI-NEXT: s_lshr_b32 s18, s56, 8
-; SI-NEXT: s_lshr_b32 s17, s40, 8
-; SI-NEXT: s_lshr_b32 s15, s22, 8
-; SI-NEXT: s_lshr_b32 s10, s28, 8
-; SI-NEXT: s_lshr_b32 s26, s16, 8
-; SI-NEXT: s_mov_b32 s42, s8
-; SI-NEXT: s_lshr_b32 s4, s8, 8
-; SI-NEXT: v_lshrrev_b32_e32 v48, 24, v1
-; SI-NEXT: v_lshrrev_b32_e32 v39, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v38, 24, v9
-; SI-NEXT: v_lshrrev_b32_e32 v35, 24, v13
-; SI-NEXT: v_lshrrev_b32_e32 v32, 24, v17
-; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v21
-; SI-NEXT: v_lshrrev_b32_e32 v28, 24, v26
-; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v25
-; SI-NEXT: s_lshr_b64 s[86:87], s[12:13], 24
-; SI-NEXT: s_lshr_b64 s[96:97], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[12:13], 8
-; SI-NEXT: s_lshr_b64 s[78:79], s[6:7], 24
-; SI-NEXT: s_lshr_b64 s[82:83], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[84:85], s[6:7], 8
-; SI-NEXT: .LBB109_5: ; %end
-; SI-NEXT: s_and_b32 s7, s74, 0xff
-; SI-NEXT: s_lshl_b32 s13, s92, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s90, 0xff
-; SI-NEXT: s_lshl_b32 s25, s88, 24
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_or_b32 s13, s25, s13
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s76, 0xff
-; SI-NEXT: s_lshl_b32 s13, s23, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s73, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v48
-; SI-NEXT: v_or_b32_e32 v2, s13, v2
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; SI-NEXT: s_lshr_b64 s[46:47], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[22:23], s[12:13], 16
+; SI-NEXT: s_mov_b32 s61, s20
+; SI-NEXT: s_lshr_b64 s[72:73], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[40:41], 16
+; SI-NEXT: s_mov_b32 s77, s92
+; SI-NEXT: s_mov_b32 s45, s74
+; SI-NEXT: s_mov_b32 s47, s78
+; SI-NEXT: s_mov_b32 s57, s26
+; SI-NEXT: s_mov_b32 s59, s24
+; SI-NEXT: s_mov_b32 s63, s22
+; SI-NEXT: s_mov_b32 s73, s64
+; SI-NEXT: s_lshr_b64 vcc, s[76:77], 8
+; SI-NEXT: s_lshr_b64 s[88:89], s[60:61], 8
+; SI-NEXT: s_lshr_b64 s[96:97], s[76:77], 24
+; SI-NEXT: s_lshr_b64 s[98:99], s[76:77], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[72:73], 24
+; SI-NEXT: s_lshr_b64 s[94:95], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[86:87], s[72:73], 8
+; SI-NEXT: s_lshr_b64 s[82:83], s[60:61], 24
+; SI-NEXT: s_lshr_b64 s[84:85], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[62:63], 24
+; SI-NEXT: s_lshr_b64 s[70:71], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[62:63], 8
+; SI-NEXT: s_lshr_b64 s[48:49], s[58:59], 24
+; SI-NEXT: s_lshr_b64 s[52:53], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[66:67], s[58:59], 8
+; SI-NEXT: s_lshr_b64 s[28:29], s[56:57], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[56:57], 8
+; SI-NEXT: s_lshr_b32 s89, s92, 8
+; SI-NEXT: s_lshr_b32 vcc_hi, s64, 8
+; SI-NEXT: s_lshr_b32 s25, s20, 8
+; SI-NEXT: s_lshr_b32 s23, s22, 8
+; SI-NEXT: s_lshr_b32 s21, s24, 8
+; SI-NEXT: s_lshr_b32 s14, s26, 8
+; SI-NEXT: s_lshr_b32 s12, s78, 8
+; SI-NEXT: s_lshr_b32 s10, s74, 8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v4
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v10
+; SI-NEXT: v_lshrrev_b32_e32 v5, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v9
+; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; SI-NEXT: s_lshr_b64 s[38:39], s[46:47], 24
+; SI-NEXT: s_lshr_b64 s[50:51], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[46:47], 8
+; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[18:19], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[44:45], 8
+; SI-NEXT: s_branch .LBB109_6
+; SI-NEXT: .LBB109_5:
+; SI-NEXT: v_readlane_b32 s16, v12, 12
+; SI-NEXT: v_mov_b32_e32 v3, s68
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: v_mov_b32_e32 v7, s99
+; SI-NEXT: v_mov_b32_e32 v8, s98
+; SI-NEXT: v_readlane_b32 s86, v12, 6
+; SI-NEXT: v_readlane_b32 s84, v12, 14
+; SI-NEXT: s_mov_b32 s4, s16
+; SI-NEXT: v_readlane_b32 s82, v12, 16
+; SI-NEXT: v_readlane_b32 s68, v12, 18
+; SI-NEXT: s_mov_b32 s28, s70
+; SI-NEXT: s_mov_b32 s16, s92
+; SI-NEXT: s_mov_b32 s18, s90
+; SI-NEXT: v_readlane_b32 s92, v12, 4
+; SI-NEXT: s_mov_b32 s70, s94
+; SI-NEXT: v_readlane_b32 s96, v12, 2
+; SI-NEXT: v_readlane_b32 s90, v12, 10
+; SI-NEXT: v_readlane_b32 s98, v12, 0
+; SI-NEXT: v_readlane_b32 s94, v12, 8
+; SI-NEXT: v_mov_b32_e32 v1, s79
+; SI-NEXT: v_mov_b32_e32 v2, s75
+; SI-NEXT: v_mov_b32_e32 v4, s42
+; SI-NEXT: v_mov_b32_e32 v5, s40
+; SI-NEXT: v_readlane_b32 s87, v12, 7
+; SI-NEXT: v_readlane_b32 s85, v12, 15
+; SI-NEXT: v_readlane_b32 s17, v12, 13
+; SI-NEXT: s_mov_b32 s26, s14
+; SI-NEXT: v_readlane_b32 s83, v12, 17
+; SI-NEXT: s_mov_b32 s24, s12
+; SI-NEXT: v_readlane_b32 s69, v12, 19
+; SI-NEXT: s_mov_b32 s22, s10
+; SI-NEXT: s_mov_b32 s64, s6
+; SI-NEXT: s_mov_b32 s20, s8
+; SI-NEXT: v_readlane_b32 s93, v12, 5
+; SI-NEXT: s_mov_b32 s88, s4
+; SI-NEXT: v_readlane_b32 s97, v12, 3
+; SI-NEXT: v_readlane_b32 s91, v12, 11
+; SI-NEXT: v_readlane_b32 s99, v12, 1
+; SI-NEXT: v_readlane_b32 s95, v12, 9
+; SI-NEXT: v_readlane_b32 s10, v12, 20
+; SI-NEXT: v_readlane_b32 s12, v12, 21
+; SI-NEXT: v_readlane_b32 s14, v12, 22
+; SI-NEXT: v_readlane_b32 s21, v12, 23
+; SI-NEXT: v_readlane_b32 s23, v12, 24
+; SI-NEXT: v_readlane_b32 s25, v12, 25
+; SI-NEXT: .LBB109_6: ; %end
+; SI-NEXT: s_and_b32 s4, s76, 0xff
+; SI-NEXT: s_lshl_b32 s6, vcc_lo, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s98, 0xff
+; SI-NEXT: s_lshl_b32 s8, s96, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: v_mov_b32_e32 v9, s4
+; SI-NEXT: s_and_b32 s4, s92, 0xff
+; SI-NEXT: s_lshl_b32 s6, s89, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s43, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; SI-NEXT: v_or_b32_e32 v8, s6, v8
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v8, s4, v8
+; SI-NEXT: s_and_b32 s4, s72, 0xff
+; SI-NEXT: s_lshl_b32 s6, s86, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s94, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s90, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_or_b32_e32 v1, s7, v2
-; SI-NEXT: s_and_b32 s7, s58, 0xff
-; SI-NEXT: s_lshl_b32 s13, s30, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, vcc_lo, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s23, s94, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s13, s23, s13
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: s_and_b32 s4, s64, 0xff
+; SI-NEXT: s_lshl_b32 s6, vcc_hi, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s41, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v7
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v7, s6, v7
+; SI-NEXT: v_or_b32_e32 v7, s4, v7
+; SI-NEXT: s_and_b32 s4, s60, 0xff
+; SI-NEXT: s_lshl_b32 s6, s88, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s84, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s82, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s62, 0xff
-; SI-NEXT: s_lshl_b32 s13, s21, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s61, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
-; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: v_mov_b32_e32 v7, s4
+; SI-NEXT: s_and_b32 s4, s20, 0xff
+; SI-NEXT: s_lshl_b32 s6, s25, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s15, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v6, s6, v6
+; SI-NEXT: v_or_b32_e32 v6, s4, v6
+; SI-NEXT: s_and_b32 s4, s62, 0xff
+; SI-NEXT: s_lshl_b32 s6, s80, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s70, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s68, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v39
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s13, v1
-; SI-NEXT: v_or_b32_e32 v1, s7, v1
-; SI-NEXT: s_and_b32 s7, s44, 0xff
-; SI-NEXT: s_lshl_b32 s13, s38, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s36, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s21, s34, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s13, s21, s13
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
+; SI-NEXT: v_mov_b32_e32 v6, s4
+; SI-NEXT: s_and_b32 s4, s22, 0xff
+; SI-NEXT: s_lshl_b32 s6, s23, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s13, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v5, s6, v5
+; SI-NEXT: v_or_b32_e32 v5, s4, v5
+; SI-NEXT: s_and_b32 s4, s58, 0xff
+; SI-NEXT: s_lshl_b32 s6, s66, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s52, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s48, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:28
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s56, 0xff
-; SI-NEXT: s_lshl_b32 s13, s18, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s47, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
-; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: v_mov_b32_e32 v5, s4
+; SI-NEXT: s_and_b32 s4, s24, 0xff
+; SI-NEXT: s_lshl_b32 s6, s21, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s11, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v4, s6, v4
+; SI-NEXT: v_or_b32_e32 v4, s4, v4
+; SI-NEXT: s_and_b32 s4, s56, 0xff
+; SI-NEXT: s_lshl_b32 s6, s36, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s34, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s28, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:36
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v38
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s13, v1
-; SI-NEXT: v_or_b32_e32 v1, s7, v1
-; SI-NEXT: s_and_b32 s7, s24, 0xff
-; SI-NEXT: s_lshl_b32 s13, s52, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s50, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s18, s48, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s13, s18, s13
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s40, 0xff
-; SI-NEXT: s_lshl_b32 s13, s17, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s27, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v35
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s13, v1
-; SI-NEXT: v_or_b32_e32 v1, s7, v1
-; SI-NEXT: s_and_b32 s7, s14, 0xff
-; SI-NEXT: s_lshl_b32 s13, s66, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s64, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s14, s54, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s22, 0xff
-; SI-NEXT: s_lshl_b32 s13, s15, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s43, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v32
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s13, v1
-; SI-NEXT: v_or_b32_e32 v1, s7, v1
-; SI-NEXT: s_and_b32 s7, s20, 0xff
-; SI-NEXT: s_lshl_b32 s13, s80, 8
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s70, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s14, s68, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s28, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 8
-; SI-NEXT: s_or_b32 s7, s7, s10
-; SI-NEXT: s_and_b32 s10, s19, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v31
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s10, v1
-; SI-NEXT: v_or_b32_e32 v1, s7, v1
-; SI-NEXT: s_and_b32 s7, s12, 0xff
-; SI-NEXT: s_lshl_b32 s10, s8, 8
-; SI-NEXT: s_or_b32 s7, s7, s10
-; SI-NEXT: s_and_b32 s10, s96, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s12, s86, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s10, s12, s10
-; SI-NEXT: s_or_b32 s7, s7, s10
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: s_and_b32 s7, s16, 0xff
-; SI-NEXT: s_lshl_b32 s9, s26, 8
-; SI-NEXT: s_or_b32 s7, s7, s9
-; SI-NEXT: s_and_b32 s9, s11, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
-; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: v_mov_b32_e32 v4, s4
+; SI-NEXT: s_and_b32 s4, s26, 0xff
+; SI-NEXT: s_lshl_b32 s6, s14, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s9, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v3, s6, v3
+; SI-NEXT: v_or_b32_e32 v3, s4, v3
+; SI-NEXT: s_and_b32 s4, s46, 0xff
+; SI-NEXT: s_lshl_b32 s6, s54, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s50, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s38, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:44
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v28
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: v_or_b32_e32 v1, s9, v1
-; SI-NEXT: v_or_b32_e32 v1, s7, v1
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s7, s84, 8
-; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_and_b32 s7, s82, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s9, s78, 24
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s9, s7
-; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
+; SI-NEXT: v_mov_b32_e32 v3, s4
+; SI-NEXT: s_and_b32 s4, s78, 0xff
+; SI-NEXT: s_lshl_b32 s6, s12, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s7, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: v_or_b32_e32 v2, s6, v2
+; SI-NEXT: v_or_b32_e32 v2, s4, v2
+; SI-NEXT: s_and_b32 s4, s44, 0xff
+; SI-NEXT: s_lshl_b32 s6, s30, 8
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s16, 24
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:52
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s6
-; SI-NEXT: s_and_b32 s6, s42, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 8
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: s_and_b32 s4, s74, 0xff
+; SI-NEXT: s_lshl_b32 s6, s10, 8
; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
-; SI-NEXT: s_or_b32 s4, s6, s4
+; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v25
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
-; SI-NEXT: v_readlane_b32 s30, v40, 34
+; SI-NEXT: v_readlane_b32 s30, v11, 34
+; SI-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen
+; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
-; SI-NEXT: v_readlane_b32 s31, v40, 35
-; SI-NEXT: v_readlane_b32 s99, v40, 33
-; SI-NEXT: v_readlane_b32 s98, v40, 32
-; SI-NEXT: v_readlane_b32 s97, v40, 31
-; SI-NEXT: v_readlane_b32 s96, v40, 30
-; SI-NEXT: v_readlane_b32 s87, v40, 29
-; SI-NEXT: v_readlane_b32 s86, v40, 28
-; SI-NEXT: v_readlane_b32 s85, v40, 27
-; SI-NEXT: v_readlane_b32 s84, v40, 26
-; SI-NEXT: v_readlane_b32 s83, v40, 25
-; SI-NEXT: v_readlane_b32 s82, v40, 24
-; SI-NEXT: v_readlane_b32 s81, v40, 23
-; SI-NEXT: v_readlane_b32 s80, v40, 22
-; SI-NEXT: v_readlane_b32 s71, v40, 21
-; SI-NEXT: v_readlane_b32 s70, v40, 20
-; SI-NEXT: v_readlane_b32 s69, v40, 19
-; SI-NEXT: v_readlane_b32 s68, v40, 18
-; SI-NEXT: v_readlane_b32 s67, v40, 17
-; SI-NEXT: v_readlane_b32 s66, v40, 16
-; SI-NEXT: v_readlane_b32 s65, v40, 15
-; SI-NEXT: v_readlane_b32 s64, v40, 14
-; SI-NEXT: v_readlane_b32 s55, v40, 13
-; SI-NEXT: v_readlane_b32 s54, v40, 12
-; SI-NEXT: v_readlane_b32 s53, v40, 11
-; SI-NEXT: v_readlane_b32 s52, v40, 10
-; SI-NEXT: v_readlane_b32 s51, v40, 9
-; SI-NEXT: v_readlane_b32 s50, v40, 8
-; SI-NEXT: v_readlane_b32 s49, v40, 7
-; SI-NEXT: v_readlane_b32 s48, v40, 6
-; SI-NEXT: v_readlane_b32 s39, v40, 5
-; SI-NEXT: v_readlane_b32 s38, v40, 4
-; SI-NEXT: v_readlane_b32 s37, v40, 3
-; SI-NEXT: v_readlane_b32 s36, v40, 2
-; SI-NEXT: v_readlane_b32 s35, v40, 1
-; SI-NEXT: v_readlane_b32 s34, v40, 0
-; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: v_readlane_b32 s31, v11, 35
+; SI-NEXT: v_readlane_b32 s99, v11, 33
+; SI-NEXT: v_readlane_b32 s98, v11, 32
+; SI-NEXT: v_readlane_b32 s97, v11, 31
+; SI-NEXT: v_readlane_b32 s96, v11, 30
+; SI-NEXT: v_readlane_b32 s87, v11, 29
+; SI-NEXT: v_readlane_b32 s86, v11, 28
+; SI-NEXT: v_readlane_b32 s85, v11, 27
+; SI-NEXT: v_readlane_b32 s84, v11, 26
+; SI-NEXT: v_readlane_b32 s83, v11, 25
+; SI-NEXT: v_readlane_b32 s82, v11, 24
+; SI-NEXT: v_readlane_b32 s81, v11, 23
+; SI-NEXT: v_readlane_b32 s80, v11, 22
+; SI-NEXT: v_readlane_b32 s71, v11, 21
+; SI-NEXT: v_readlane_b32 s70, v11, 20
+; SI-NEXT: v_readlane_b32 s69, v11, 19
+; SI-NEXT: v_readlane_b32 s68, v11, 18
+; SI-NEXT: v_readlane_b32 s67, v11, 17
+; SI-NEXT: v_readlane_b32 s66, v11, 16
+; SI-NEXT: v_readlane_b32 s65, v11, 15
+; SI-NEXT: v_readlane_b32 s64, v11, 14
+; SI-NEXT: v_readlane_b32 s55, v11, 13
+; SI-NEXT: v_readlane_b32 s54, v11, 12
+; SI-NEXT: v_readlane_b32 s53, v11, 11
+; SI-NEXT: v_readlane_b32 s52, v11, 10
+; SI-NEXT: v_readlane_b32 s51, v11, 9
+; SI-NEXT: v_readlane_b32 s50, v11, 8
+; SI-NEXT: v_readlane_b32 s49, v11, 7
+; SI-NEXT: v_readlane_b32 s48, v11, 6
+; SI-NEXT: v_readlane_b32 s39, v11, 5
+; SI-NEXT: v_readlane_b32 s38, v11, 4
+; SI-NEXT: v_readlane_b32 s37, v11, 3
+; SI-NEXT: v_readlane_b32 s36, v11, 2
+; SI-NEXT: v_readlane_b32 s35, v11, 1
+; SI-NEXT: v_readlane_b32 s34, v11, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -91422,741 +90186,735 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:48
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:64
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:48
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:88
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:96
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:112
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:80
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:128
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:108
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:104
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:28
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:112
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:72
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:104
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:84
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:76
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v7
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v11
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v13
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v15
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v11
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; kill: killed $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: v_lshlrev_b32_e32 v38, 8, v13
-; SI-NEXT: v_lshlrev_b32_e32 v36, 24, v19
-; SI-NEXT: v_lshlrev_b32_e32 v51, 8, v21
-; SI-NEXT: v_lshlrev_b32_e32 v48, 24, v23
-; SI-NEXT: v_lshlrev_b32_e32 v49, 24, v27
-; SI-NEXT: v_lshlrev_b32_e32 v42, 8, v29
-; SI-NEXT: ; kill: killed $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; kill: killed $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v23
+; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v7
+; SI-NEXT: v_lshlrev_b32_e32 v55, 8, v21
+; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v19
+; SI-NEXT: v_lshlrev_b32_e32 v56, 8, v29
+; SI-NEXT: v_lshlrev_b32_e32 v44, 24, v27
+; SI-NEXT: v_lshlrev_b32_e32 v47, 8, v25
+; SI-NEXT: ; kill: killed $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; kill: killed $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr44
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: ; implicit-def: $vgpr43
; SI-NEXT: ; implicit-def: $vgpr45
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr62
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr49
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v1
-; SI-NEXT: v_lshlrev_b32_e32 v40, 24, v2
-; SI-NEXT: v_lshlrev_b32_e32 v60, 8, v6
-; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v8
-; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v9
-; SI-NEXT: v_lshlrev_b32_e32 v22, 8, v14
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v60, 8, v4
+; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v6
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v62, 8, v12
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v16
-; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v17
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v18
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v9
+; SI-NEXT: v_lshlrev_b32_e32 v63, 24, v16
+; SI-NEXT: v_lshlrev_b32_e32 v42, 24, v2
; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v20
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v24
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v25
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:40
+; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v26
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v28
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:60
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:44
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v14
+; SI-NEXT: v_lshlrev_b32_e32 v18, 8, v18
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v22
+; SI-NEXT: v_lshlrev_b32_e32 v27, 24, v31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v17
+; SI-NEXT: v_lshlrev_b32_e32 v31, 8, v32
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v28
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:20
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:28
+; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v33
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v35
+; SI-NEXT: v_lshlrev_b32_e32 v51, 8, v36
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:100
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v11, 0xff, v26
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
-; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xff, v52
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_or_b32_e32 v16, v16, v13
+; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT: v_or_b32_e32 v1, v6, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v16
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_or_b32_e32 v39, v46, v9
+; SI-NEXT: ; implicit-def: $vgpr46
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v29, v5, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v23, v0, v29
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v39
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v11, v11, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v14, v5, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_or_b32_e32 v21, v14, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
-; SI-NEXT: v_or_b32_e32 v23, v13, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_or_b32_e32 v3, v3, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v27, v13, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v36, v5, v0
+; SI-NEXT: v_alignbit_b32 v0, v23, v14, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_or_b32_e32 v3, v0, v36
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v36
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_or_b32_e32 v11, v11, v38
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v50, v13, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v17, v5, v0
+; SI-NEXT: v_alignbit_b32 v0, v3, v17, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
-; SI-NEXT: v_or_b32_e32 v52, v13, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT: v_or_b32_e32 v0, v0, v55
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v37, v7, v5
+; SI-NEXT: v_or_b32_e32 v5, v0, v37
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v37
+; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v53, v36, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v19, v54, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: v_or_b32_e32 v38, v42, v7
+; SI-NEXT: v_alignbit_b32 v53, v5, v19, 16
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_or_b32_e32 v11, v11, v51
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v56
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v7, v0, v38
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v38
+; SI-NEXT: ; implicit-def: $vgpr56
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v41, v48, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v25, v44, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v45, v7, v25, 16
+; SI-NEXT: ; implicit-def: $vgpr44
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v44, v13, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v60
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v9, v0, v39
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v20
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v20, v57, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v12
+; SI-NEXT: v_or_b32_e32 v0, v0, v62
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v12, v59, v11
+; SI-NEXT: v_or_b32_e32 v11, v0, v12
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v28
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v26, v63, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v40
+; SI-NEXT: v_or_b32_e32 v0, v0, v18
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v13, v0, v16
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v41
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v2, v2, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v30
+; SI-NEXT: v_or_b32_e32 v0, v0, v34
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v4, v4, v8
+; SI-NEXT: v_or_b32_e32 v15, v0, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v24
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v8, v27, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v32, v9, v20, 16
+; SI-NEXT: v_alignbit_b32 v22, v11, v26, 16
+; SI-NEXT: v_alignbit_b32 v21, v13, v2, 16
+; SI-NEXT: v_alignbit_b32 v33, v15, v8, 16
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v4
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr62
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr63
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v45, v49, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v0, v0, v18
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v13
-; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_or_b32_e32 v11, v11, v42
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: v_or_b32_e32 v14, v14, v18
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_or_b32_e32 v14, v14, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v57, v54, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: v_or_b32_e32 v17, v17, v18
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v61, v17, v19
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v19, v1, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v58
+; SI-NEXT: v_or_b32_e32 v1, v1, v51
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v58, v13, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v13, 24, v25
-; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT: v_or_b32_e32 v17, v17, v47
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v43, v17, v25
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v61, v40, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v37
-; SI-NEXT: v_or_b32_e32 v11, v11, v60
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT: v_or_b32_e32 v17, v17, v31
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v48, v17, v20
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v31, v47, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v39
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v32, v13, v11
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v20
-; SI-NEXT: v_or_b32_e32 v13, v13, v22
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v17
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_or_b32_e32 v13, v3, v13
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v33, v0, v3
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v30, v26, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v7
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v43
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v24, v16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v63
-; SI-NEXT: v_or_b32_e32 v28, v3, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v56
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v15, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v12
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v19, v8, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT: v_or_b32_e32 v10, v17, v10
+; SI-NEXT: v_or_b32_e32 v17, v1, v8
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v18
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_or_b32_e32 v11, v59, v11
-; SI-NEXT: ; kill: killed $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr63
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_or_b32_e32 v25, v10, v26
+; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; kill: killed $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: .LBB110_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v12
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v11
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0x300, v2
-; SI-NEXT: v_or_b32_e32 v4, v8, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v2, v4, v2
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v10
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v8, 8, v63
-; SI-NEXT: v_or_b32_e32 v4, v8, v4
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v56
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v58
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v24
+; SI-NEXT: v_or_b32_e32 v0, v51, v0
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_i32_e32 v0, vcc, 0x300, v0
+; SI-NEXT: v_or_b32_e32 v3, v27, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v3, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v8
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_or_b32_e32 v3, v34, v3
; SI-NEXT: s_movk_i32 s6, 0x300
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v4
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_or_b32_e32 v1, v1, v8
-; SI-NEXT: v_or_b32_e32 v1, v1, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v43
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_add_i32_e32 v4, vcc, s6, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_or_b32_e32 v6, v16, v6
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v9
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT: v_or_b32_e32 v0, v0, v6
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT: s_mov_b32 s7, 0x3000000
+; SI-NEXT: v_add_i32_e32 v17, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v3
+; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v30
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v3, v4, v3
+; SI-NEXT: v_or_b32_e32 v0, v3, v0
+; SI-NEXT: v_add_i32_e32 v15, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v6
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v41
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v5, v26, v5
-; SI-NEXT: v_or_b32_e32 v0, v5, v0
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v20
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v17
-; SI-NEXT: v_or_b32_e32 v5, v22, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_or_b32_e32 v3, v3, v6
-; SI-NEXT: v_or_b32_e32 v3, v3, v5
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v39
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v25
-; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v18
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT: v_or_b32_e32 v6, v59, v6
-; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: v_add_i32_e32 v6, vcc, 3, v37
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT: v_or_b32_e32 v6, v60, v6
-; SI-NEXT: v_add_i32_e32 v6, vcc, s6, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: s_mov_b32 s7, 0x3000000
-; SI-NEXT: v_add_i32_e32 v2, vcc, s7, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT: v_add_i32_e32 v4, vcc, s7, v4
-; SI-NEXT: v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT: v_add_i32_e32 v3, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, s7, v5
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v5
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_or_b32_e32 v7, v47, v7
-; SI-NEXT: v_or_b32_e32 v6, v7, v6
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v8, 8, v8
-; SI-NEXT: v_add_i32_e32 v6, vcc, s7, v6
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v62, 16, v6
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v0
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v1
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v2
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_or_b32_e32 v9, v54, v9
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_or_b32_e32 v11, v48, v11
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v12, 8, v12
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT: v_or_b32_e32 v7, v8, v7
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_or_b32_e32 v8, v40, v8
-; SI-NEXT: v_or_b32_e32 v7, v8, v7
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v7, vcc, s7, v7
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v7
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 8, v14
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT: v_or_b32_e32 v8, v42, v8
-; SI-NEXT: v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_or_b32_e32 v8, v9, v8
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v8, vcc, s7, v8
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT: v_or_b32_e32 v9, v10, v9
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v9, vcc, s6, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v19, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v40
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v52
+; SI-NEXT: v_or_b32_e32 v0, v18, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v16, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v13, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v28
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v63, v1
+; SI-NEXT: v_alignbit_b32 v21, v13, v19, 16
+; SI-NEXT: v_alignbit_b32 v33, v15, v17, 16
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v15
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_or_b32_e32 v0, v10, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v25, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v12
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v26
+; SI-NEXT: v_or_b32_e32 v0, v62, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v59, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v11, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v20
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v57, v1
+; SI-NEXT: v_alignbit_b32 v22, v11, v25, 16
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v11
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_or_b32_e32 v10, v49, v10
-; SI-NEXT: v_or_b32_e32 v9, v10, v9
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v9, vcc, s7, v9
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v9
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_or_b32_e32 v0, v31, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v48, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT: v_or_b32_e32 v10, v51, v10
-; SI-NEXT: v_add_i32_e32 v10, vcc, s6, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_or_b32_e32 v10, v11, v10
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v10, vcc, s7, v10
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v55, 16, v10
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_or_b32_e32 v0, v60, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v46, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v9, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v32, v9, v48, 16
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v9
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_or_b32_e32 v11, v12, v11
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v11, vcc, s6, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_or_b32_e32 v0, v47, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v44, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v43, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_or_b32_e32 v12, v36, v12
-; SI-NEXT: v_or_b32_e32 v11, v12, v11
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v11, vcc, s7, v11
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v5
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_or_b32_e32 v0, v56, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v42, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v7, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v45, v7, v43, 16
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v7
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT: v_or_b32_e32 v12, v38, v12
-; SI-NEXT: v_add_i32_e32 v12, vcc, s6, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_or_b32_e32 v12, v13, v12
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v12, vcc, s7, v12
-; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v12
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v13, vcc, s6, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v54, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v61, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: v_or_b32_e32 v13, v14, v13
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v13, vcc, s7, v13
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_or_b32_e32 v0, v55, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v5, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: v_alignbit_b32 v53, v5, v61, 16
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v5
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xff, v14
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v14, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_or_b32_e32 v15, v16, v15
-; SI-NEXT: v_or_b32_e32 v14, v15, v14
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v14, vcc, s7, v14
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v3
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, s7, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v3
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 8, v16
-; SI-NEXT: v_or_b32_e32 v15, v16, v15
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT: v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_or_b32_e32 v16, v17, v16
-; SI-NEXT: v_or_b32_e32 v15, v16, v15
-; SI-NEXT: v_add_i32_e32 v15, vcc, s7, v15
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v0, vcc, s7, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v4, v2
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_add_i32_e32 v23, vcc, s7, v1
+; SI-NEXT: v_alignbit_b32 v1, v23, v0, 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: v_alignbit_b32 v1, v3, v14, 16
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v23
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: .LBB110_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v23
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v29
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v55
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v44
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v46
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v58
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v62
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v14
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v34
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v35
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v35
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v53
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v45
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v32
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v22
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v33
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v23
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v14
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v21
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v27
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v50
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v53
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v41
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_alignbit_b32 v5, v5, v6, 16
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v45
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_alignbit_b32 v6, v6, v7, 16
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v57
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_alignbit_b32 v7, v7, v8, 16
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v61
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v36
+; SI-NEXT: v_or_b32_e32 v3, v3, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v61
+; SI-NEXT: v_or_b32_e32 v4, v4, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v37
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v43
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
@@ -92173,32 +90931,26 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_alignbit_b32 v8, v8, v9, 16
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v31
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; SI-NEXT: v_alignbit_b32 v9, v9, v10, 16
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v32
-; SI-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v13
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v30
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v33
-; SI-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v24
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v28
-; SI-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v19
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v38
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v48
+; SI-NEXT: v_or_b32_e32 v8, v8, v10
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v29
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v25
+; SI-NEXT: v_or_b32_e32 v10, v10, v12
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v39
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v21
+; SI-NEXT: v_or_b32_e32 v12, v12, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v49
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v17
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v50
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -93936,8 +92688,11 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; SI-NEXT: v_writelane_b32 v43, s86, 28
; SI-NEXT: v_writelane_b32 v43, s87, 29
; SI-NEXT: v_writelane_b32 v43, s96, 30
-; SI-NEXT: v_writelane_b32 v43, s30, 31
-; SI-NEXT: v_writelane_b32 v43, s31, 32
+; SI-NEXT: v_writelane_b32 v43, s97, 31
+; SI-NEXT: v_writelane_b32 v43, s98, 32
+; SI-NEXT: v_writelane_b32 v43, s99, 33
+; SI-NEXT: v_writelane_b32 v43, s30, 34
+; SI-NEXT: v_writelane_b32 v43, s31, 35
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:76
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
@@ -93961,577 +92716,635 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32
+; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: ; implicit-def: $vgpr44 : SGPR spill to VGPR lane
-; SI-NEXT: v_readfirstlane_b32 s9, v1
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: v_readfirstlane_b32 s7, v4
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_readfirstlane_b32 s89, v30
-; SI-NEXT: v_readfirstlane_b32 s63, v29
-; SI-NEXT: v_readfirstlane_b32 s72, v28
-; SI-NEXT: v_readfirstlane_b32 s75, v27
-; SI-NEXT: v_readfirstlane_b32 s77, v26
-; SI-NEXT: v_readfirstlane_b32 s59, v25
-; SI-NEXT: v_readfirstlane_b32 s60, v24
-; SI-NEXT: v_readfirstlane_b32 s62, v23
-; SI-NEXT: v_readfirstlane_b32 s73, v22
-; SI-NEXT: v_readfirstlane_b32 s47, v21
-; SI-NEXT: v_readfirstlane_b32 s56, v20
-; SI-NEXT: v_readfirstlane_b32 s58, v19
-; SI-NEXT: v_readfirstlane_b32 s61, v18
-; SI-NEXT: v_readfirstlane_b32 s43, v17
-; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
+; SI-NEXT: v_readfirstlane_b32 s74, v0
+; SI-NEXT: v_writelane_b32 v44, s7, 0
+; SI-NEXT: v_readfirstlane_b32 s9, v9
+; SI-NEXT: v_writelane_b32 v44, s74, 1
+; SI-NEXT: v_readfirstlane_b32 s56, v8
+; SI-NEXT: v_writelane_b32 v44, s9, 2
+; SI-NEXT: v_readfirstlane_b32 s58, v7
+; SI-NEXT: v_writelane_b32 v44, s56, 3
+; SI-NEXT: v_readfirstlane_b32 s72, v6
+; SI-NEXT: v_writelane_b32 v44, s58, 4
+; SI-NEXT: v_readfirstlane_b32 s8, v5
+; SI-NEXT: v_writelane_b32 v44, s72, 5
+; SI-NEXT: v_readfirstlane_b32 s10, v4
+; SI-NEXT: v_writelane_b32 v44, s8, 6
+; SI-NEXT: v_readfirstlane_b32 s11, v17
+; SI-NEXT: v_writelane_b32 v44, s10, 7
+; SI-NEXT: v_readfirstlane_b32 s46, v16
+; SI-NEXT: v_writelane_b32 v44, s11, 8
+; SI-NEXT: v_readfirstlane_b32 s60, v2
+; SI-NEXT: v_writelane_b32 v44, s46, 9
+; SI-NEXT: v_readfirstlane_b32 s63, v30
+; SI-NEXT: v_readfirstlane_b32 s52, v29
+; SI-NEXT: v_readfirstlane_b32 s51, v28
+; SI-NEXT: v_readfirstlane_b32 s53, v27
+; SI-NEXT: v_readfirstlane_b32 s54, v26
+; SI-NEXT: v_readfirstlane_b32 s34, v25
+; SI-NEXT: v_readfirstlane_b32 s61, v24
+; SI-NEXT: v_readfirstlane_b32 s37, v23
+; SI-NEXT: v_readfirstlane_b32 s38, v22
+; SI-NEXT: v_readfirstlane_b32 s48, v21
+; SI-NEXT: v_readfirstlane_b32 s39, v20
+; SI-NEXT: v_readfirstlane_b32 s49, v19
+; SI-NEXT: v_readfirstlane_b32 s50, v18
+; SI-NEXT: v_readfirstlane_b32 s47, v15
; SI-NEXT: v_readfirstlane_b32 s57, v14
-; SI-NEXT: v_readfirstlane_b32 s15, v13
-; SI-NEXT: v_readfirstlane_b32 s40, v12
-; SI-NEXT: v_readfirstlane_b32 s42, v11
-; SI-NEXT: v_readfirstlane_b32 s45, v10
-; SI-NEXT: v_readfirstlane_b32 s5, v9
-; SI-NEXT: v_readfirstlane_b32 s87, v8
-; SI-NEXT: v_readfirstlane_b32 s14, v7
-; SI-NEXT: v_readfirstlane_b32 s41, v6
-; SI-NEXT: v_readfirstlane_b32 s13, v2
+; SI-NEXT: v_readfirstlane_b32 s30, v13
+; SI-NEXT: v_readfirstlane_b32 s59, v12
+; SI-NEXT: v_readfirstlane_b32 s35, v11
+; SI-NEXT: v_readfirstlane_b32 s36, v10
+; SI-NEXT: v_readfirstlane_b32 s45, v3
+; SI-NEXT: v_writelane_b32 v44, s60, 10
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT: s_and_b64 s[10:11], vcc, exec
-; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: v_writelane_b32 v44, s10, 0
-; SI-NEXT: v_writelane_b32 v44, s9, 1
-; SI-NEXT: v_writelane_b32 v44, s6, 2
-; SI-NEXT: v_writelane_b32 v44, s7, 3
-; SI-NEXT: v_readfirstlane_b32 s55, v32
-; SI-NEXT: v_readfirstlane_b32 s54, v33
-; SI-NEXT: v_readfirstlane_b32 s64, v34
-; SI-NEXT: v_readfirstlane_b32 s65, v35
-; SI-NEXT: v_readfirstlane_b32 s51, v36
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: v_readfirstlane_b32 s12, v32
+; SI-NEXT: v_readfirstlane_b32 s14, v33
+; SI-NEXT: v_readfirstlane_b32 s40, v34
+; SI-NEXT: v_readfirstlane_b32 s41, v35
+; SI-NEXT: v_readfirstlane_b32 s13, v36
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_readfirstlane_b32 s50, v37
+; SI-NEXT: v_readfirstlane_b32 s15, v37
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_readfirstlane_b32 s52, v38
+; SI-NEXT: v_readfirstlane_b32 s5, v38
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_readfirstlane_b32 s53, v39
+; SI-NEXT: v_readfirstlane_b32 s76, v39
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s39, v48
+; SI-NEXT: v_readfirstlane_b32 s31, v48
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s38, v49
+; SI-NEXT: v_readfirstlane_b32 s73, v49
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s48, v50
+; SI-NEXT: v_readfirstlane_b32 s75, v50
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_readfirstlane_b32 s49, v51
+; SI-NEXT: v_readfirstlane_b32 s44, v51
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s35, v52
+; SI-NEXT: v_readfirstlane_b32 s42, v52
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s34, v53
+; SI-NEXT: v_readfirstlane_b32 s43, v53
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s36, v54
+; SI-NEXT: v_readfirstlane_b32 s65, v54
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s37, v55
+; SI-NEXT: v_readfirstlane_b32 s66, v55
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s30, v40
+; SI-NEXT: v_readfirstlane_b32 s55, v40
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s4, v41
+; SI-NEXT: v_readfirstlane_b32 s82, v41
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s31, v42
-; SI-NEXT: v_writelane_b32 v44, s8, 4
-; SI-NEXT: v_writelane_b32 v44, s5, 5
+; SI-NEXT: v_readfirstlane_b32 s64, v42
+; SI-NEXT: v_writelane_b32 v44, s76, 11
; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s11, s16, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s12, s17, 24
-; SI-NEXT: s_or_b32 s74, s12, s11
-; SI-NEXT: s_and_b32 s11, s18, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s12, s19, 24
-; SI-NEXT: s_or_b32 s76, s12, s11
-; SI-NEXT: s_and_b32 s11, s20, 0xff
-; SI-NEXT: s_lshl_b32 s12, s21, 8
-; SI-NEXT: s_or_b32 s12, s11, s12
-; SI-NEXT: s_and_b32 s11, s22, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s78, s23, 24
-; SI-NEXT: s_or_b32 s79, s78, s11
-; SI-NEXT: s_and_b32 s11, s24, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s78, s25, 24
+; SI-NEXT: s_and_b32 s4, s16, 0xff
+; SI-NEXT: s_lshl_b32 s6, s17, 8
+; SI-NEXT: s_or_b32 vcc_hi, s4, s6
+; SI-NEXT: s_and_b32 s4, s18, 0xff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s6, s19, 24
+; SI-NEXT: s_or_b32 s4, s6, s4
+; SI-NEXT: s_and_b32 s6, s24, 0xff
+; SI-NEXT: s_lshl_b32 s62, s25, 8
+; SI-NEXT: s_or_b32 s67, s6, s62
+; SI-NEXT: s_and_b32 s6, s26, 0xff
+; SI-NEXT: s_and_b32 s72, s72, 0xff
+; SI-NEXT: s_lshl_b32 s58, s58, 8
+; SI-NEXT: s_and_b32 s56, s56, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s62, s27, 24
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: s_or_b32 s78, s78, s11
-; SI-NEXT: s_and_b32 s11, s26, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s58, s72, s58
+; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: s_lshl_b32 s9, s9, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s88, s27, 24
-; SI-NEXT: s_or_b32 s91, s9, s10
-; SI-NEXT: s_and_b32 s9, s13, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s6, s6, 24
-; SI-NEXT: s_or_b32 s88, s88, s11
-; SI-NEXT: s_and_b32 s11, s28, 0xff
-; SI-NEXT: s_lshl_b32 s90, s29, 8
-; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_or_b32 s6, s62, s6
+; SI-NEXT: s_and_b32 s60, s60, 0xff
+; SI-NEXT: s_lshl_b32 s62, s45, 8
+; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
-; SI-NEXT: s_or_b32 s92, s6, s7
-; SI-NEXT: s_and_b32 s6, s41, 0xff
-; SI-NEXT: s_lshl_b32 s7, s14, 8
-; SI-NEXT: s_or_b32 s67, s11, s90
-; SI-NEXT: s_or_b32 s90, s8, s9
-; SI-NEXT: s_or_b32 s8, s6, s7
-; SI-NEXT: s_and_b32 s6, s87, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s5, s5, 24
-; SI-NEXT: s_or_b32 s94, s5, s6
-; SI-NEXT: s_and_b32 s5, s45, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s42, 24
-; SI-NEXT: s_or_b32 s93, s6, s5
-; SI-NEXT: s_and_b32 s5, s40, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s15, 24
-; SI-NEXT: s_or_b32 s95, s6, s5
-; SI-NEXT: s_and_b32 s5, s57, 0xff
-; SI-NEXT: s_lshl_b32 s6, s46, 8
-; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_and_b32 s6, s44, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s43, 24
-; SI-NEXT: s_or_b32 vcc_hi, s7, s6
-; SI-NEXT: s_and_b32 s6, s61, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s58, 24
-; SI-NEXT: s_or_b32 vcc_lo, s7, s6
-; SI-NEXT: s_and_b32 s6, s56, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s47, 24
-; SI-NEXT: s_or_b32 s86, s7, s6
-; SI-NEXT: s_and_b32 s6, s73, 0xff
-; SI-NEXT: s_lshl_b32 s7, s62, 8
-; SI-NEXT: s_or_b32 s10, s6, s7
-; SI-NEXT: s_and_b32 s6, s60, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s59, 24
-; SI-NEXT: s_or_b32 s85, s7, s6
-; SI-NEXT: s_and_b32 s6, s77, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s75, 24
-; SI-NEXT: s_or_b32 s83, s7, s6
-; SI-NEXT: s_and_b32 s6, s72, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s63, 24
-; SI-NEXT: s_or_b32 s84, s7, s6
-; SI-NEXT: s_and_b32 s6, s89, 0xff
-; SI-NEXT: s_lshl_b32 s7, s31, 8
-; SI-NEXT: s_or_b32 s81, s6, s7
-; SI-NEXT: s_mov_b32 s6, s4
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s7, s30, 24
-; SI-NEXT: s_or_b32 s82, s7, s4
-; SI-NEXT: s_and_b32 s4, s37, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s7, s36, 24
-; SI-NEXT: s_or_b32 s71, s7, s4
-; SI-NEXT: s_and_b32 s4, s34, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s7, s35, 24
-; SI-NEXT: s_or_b32 s80, s7, s4
-; SI-NEXT: s_and_b32 s4, s49, 0xff
-; SI-NEXT: s_lshl_b32 s7, s48, 8
-; SI-NEXT: s_or_b32 s4, s4, s7
-; SI-NEXT: s_and_b32 s7, s38, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s9, s39, 24
-; SI-NEXT: s_or_b32 s68, s9, s7
-; SI-NEXT: s_and_b32 s7, s53, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s9, s52, 24
-; SI-NEXT: s_or_b32 s7, s9, s7
-; SI-NEXT: s_and_b32 s9, s50, 0xff
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s11, s51, 24
-; SI-NEXT: s_or_b32 s66, s11, s9
-; SI-NEXT: s_and_b32 s9, s65, 0xff
-; SI-NEXT: s_lshl_b32 s11, s64, 8
-; SI-NEXT: s_or_b32 s96, s9, s11
-; SI-NEXT: s_and_b32 s9, s54, 0xff
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s11, s55, 24
-; SI-NEXT: s_or_b32 s11, s11, s9
-; SI-NEXT: s_lshl_b32 s69, s12, 16
-; SI-NEXT: s_lshl_b32 s70, s67, 16
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s5, 16
+; SI-NEXT: s_and_b32 s58, s58, 0xffff
+; SI-NEXT: s_or_b32 s87, s9, s56
+; SI-NEXT: s_or_b32 s70, s60, s62
+; SI-NEXT: s_or_b32 s8, s8, s10
+; SI-NEXT: s_and_b32 s10, s36, 0xff
+; SI-NEXT: s_lshl_b32 s60, s35, 8
+; SI-NEXT: s_or_b32 s9, s58, s87
+; SI-NEXT: s_and_b32 s56, s57, 0xff
+; SI-NEXT: s_lshl_b32 s58, s47, 8
+; SI-NEXT: s_and_b32 s46, s46, 0xff
+; SI-NEXT: s_or_b32 s71, s10, s60
+; SI-NEXT: s_and_b32 s10, s59, 0xff
+; SI-NEXT: s_or_b32 s56, s56, s58
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s81, s81, 16
-; SI-NEXT: s_lshl_b32 s67, s4, 16
-; SI-NEXT: s_lshl_b32 s12, s96, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshl_b32 s60, s30, 24
+; SI-NEXT: s_and_b32 s56, s56, 0xffff
+; SI-NEXT: s_or_b32 s96, s11, s46
+; SI-NEXT: s_or_b32 s10, s60, s10
+; SI-NEXT: s_and_b32 s60, s50, 0xff
+; SI-NEXT: s_lshl_b32 s62, s49, 8
+; SI-NEXT: s_or_b32 s11, s56, s96
+; SI-NEXT: s_and_b32 s46, s38, 0xff
+; SI-NEXT: s_lshl_b32 s56, s37, 8
+; SI-NEXT: s_or_b32 s80, s60, s62
+; SI-NEXT: s_and_b32 s60, s39, 0xff
+; SI-NEXT: s_or_b32 s46, s46, s56
+; SI-NEXT: s_and_b32 s56, s61, 0xff
+; SI-NEXT: s_lshl_b32 s60, s60, 16
+; SI-NEXT: s_lshl_b32 s62, s48, 24
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_lshl_b32 s58, s34, 24
+; SI-NEXT: s_or_b32 s60, s62, s60
+; SI-NEXT: s_and_b32 s62, s54, 0xff
+; SI-NEXT: s_lshl_b32 s77, s53, 8
+; SI-NEXT: s_and_b32 s46, s46, 0xffff
+; SI-NEXT: s_or_b32 s97, s58, s56
+; SI-NEXT: s_or_b32 s81, s62, s77
+; SI-NEXT: s_and_b32 s62, s51, 0xff
+; SI-NEXT: s_and_b32 s76, s76, 0xff
+; SI-NEXT: s_mov_b32 s79, s5
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_mov_b32 s72, s61
+; SI-NEXT: s_or_b32 s61, s46, s97
+; SI-NEXT: s_and_b32 s46, s63, 0xff
+; SI-NEXT: s_lshl_b32 s56, s64, 8
+; SI-NEXT: s_lshl_b32 s62, s62, 16
+; SI-NEXT: s_lshl_b32 s77, s52, 24
+; SI-NEXT: s_or_b32 s84, s76, s5
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s76, s21, 8
+; SI-NEXT: s_or_b32 s46, s46, s56
+; SI-NEXT: s_and_b32 s56, s82, 0xff
+; SI-NEXT: s_or_b32 s62, s77, s62
+; SI-NEXT: s_and_b32 s77, s66, 0xff
+; SI-NEXT: s_lshl_b32 s78, s65, 8
+; SI-NEXT: s_or_b32 s5, s5, s76
+; SI-NEXT: s_and_b32 s76, s22, 0xff
+; SI-NEXT: s_and_b32 s74, s74, 0xff
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_lshl_b32 s58, s55, 24
+; SI-NEXT: s_mov_b32 s91, s14
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_or_b32 s83, s77, s78
+; SI-NEXT: s_lshl_b32 s76, s76, 16
+; SI-NEXT: s_lshl_b32 s77, s23, 24
+; SI-NEXT: s_lshl_b32 s74, s74, 16
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_and_b32 s46, s46, 0xffff
+; SI-NEXT: s_or_b32 s98, s58, s56
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_mov_b32 s94, s12
+; SI-NEXT: s_lshl_b32 s12, s12, 24
+; SI-NEXT: s_or_b32 s85, s77, s76
+; SI-NEXT: s_and_b32 s76, s28, 0xff
+; SI-NEXT: s_lshl_b32 s77, s29, 8
+; SI-NEXT: s_or_b32 s86, s7, s74
+; SI-NEXT: s_mov_b32 s74, s63
+; SI-NEXT: s_or_b32 s63, s46, s98
+; SI-NEXT: s_mov_b32 s88, s44
+; SI-NEXT: s_and_b32 s44, s44, 0xff
+; SI-NEXT: s_lshl_b32 s46, s75, 8
+; SI-NEXT: s_mov_b32 s92, s41
+; SI-NEXT: s_and_b32 s41, s41, 0xff
+; SI-NEXT: s_mov_b32 s93, s40
+; SI-NEXT: s_lshl_b32 s40, s40, 8
+; SI-NEXT: s_or_b32 s78, s12, s14
+; SI-NEXT: s_and_b32 s12, s15, 0xff
+; SI-NEXT: s_or_b32 s76, s76, s77
+; SI-NEXT: s_or_b32 s44, s44, s46
+; SI-NEXT: s_and_b32 s46, s73, 0xff
+; SI-NEXT: s_mov_b32 s89, s43
+; SI-NEXT: s_and_b32 s43, s43, 0xff
+; SI-NEXT: s_or_b32 s40, s41, s40
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_mov_b32 vcc_lo, s13
+; SI-NEXT: s_lshl_b32 s13, s13, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_lshl_b32 s56, s31, 24
+; SI-NEXT: s_lshl_b32 s43, s43, 16
+; SI-NEXT: s_mov_b32 s90, s42
+; SI-NEXT: s_lshl_b32 s42, s42, 24
+; SI-NEXT: s_or_b32 s68, s13, s12
+; SI-NEXT: s_and_b32 s12, s40, 0xffff
+; SI-NEXT: s_or_b32 s5, s5, s85
+; SI-NEXT: s_or_b32 s7, s76, s86
+; SI-NEXT: s_or_b32 s99, s56, s46
+; SI-NEXT: s_or_b32 s76, s42, s43
+; SI-NEXT: s_and_b32 s42, s44, 0xffff
+; SI-NEXT: s_or_b32 s69, s12, s78
+; SI-NEXT: s_and_b32 s12, vcc_hi, 0xffff
+; SI-NEXT: s_or_b32 s77, s42, s99
+; SI-NEXT: s_or_b32 s42, s12, s4
+; SI-NEXT: s_mov_b32 s12, s45
+; SI-NEXT: s_lshr_b64 s[44:45], s[4:5], 16
+; SI-NEXT: s_mov_b32 s4, s47
+; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
+; SI-NEXT: s_and_b32 s13, s67, 0xffff
+; SI-NEXT: s_and_b32 s14, s70, 0xffff
+; SI-NEXT: s_and_b32 s58, s71, 0xffff
+; SI-NEXT: s_and_b32 vcc_hi, s80, 0xffff
+; SI-NEXT: s_and_b32 s67, s81, 0xffff
+; SI-NEXT: s_mov_b32 s47, s4
+; SI-NEXT: s_mov_b32 s4, s57
+; SI-NEXT: s_lshr_b64 s[56:57], s[8:9], 16
+; SI-NEXT: s_mov_b32 s95, s15
+; SI-NEXT: s_and_b32 s70, s83, 0xffff
+; SI-NEXT: s_mov_b32 s45, s12
+; SI-NEXT: s_or_b32 s40, s13, s6
+; SI-NEXT: s_or_b32 s14, s14, s8
+; SI-NEXT: s_mov_b32 s15, s9
+; SI-NEXT: s_mov_b32 s57, s4
+; SI-NEXT: s_or_b32 s12, s58, s10
+; SI-NEXT: s_mov_b32 s13, s11
+; SI-NEXT: s_mov_b32 s4, s59
+; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
+; SI-NEXT: s_or_b32 s10, vcc_hi, s60
+; SI-NEXT: s_mov_b32 s11, s61
+; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
+; SI-NEXT: s_or_b32 s8, s67, s62
+; SI-NEXT: s_mov_b32 s9, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
+; SI-NEXT: s_and_b32 s71, s84, 0xffff
+; SI-NEXT: s_mov_b32 s59, s4
+; SI-NEXT: s_mov_b32 s61, s72
+; SI-NEXT: s_mov_b32 s63, s74
+; SI-NEXT: s_or_b32 s6, s70, s76
+; SI-NEXT: s_mov_b32 s4, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[76:77], 16
+; SI-NEXT: s_mov_b32 s76, s75
+; SI-NEXT: s_lshr_b64 s[74:75], s[68:69], 16
+; SI-NEXT: s_mov_b32 s43, s5
+; SI-NEXT: s_mov_b32 s41, s7
+; SI-NEXT: s_mov_b32 s7, s77
+; SI-NEXT: s_mov_b32 s73, s4
+; SI-NEXT: s_or_b32 s4, s71, s68
+; SI-NEXT: s_mov_b32 s5, s69
+; SI-NEXT: s_mov_b32 s75, s76
+; SI-NEXT: s_lshr_b32 s71, s85, 16
+; SI-NEXT: s_lshr_b32 s70, s86, 16
+; SI-NEXT: s_lshr_b32 s68, s87, 16
+; SI-NEXT: s_lshr_b32 s69, s96, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s97, 16
+; SI-NEXT: s_lshr_b32 s67, s98, 16
+; SI-NEXT: s_lshr_b32 s80, s99, 16
+; SI-NEXT: s_lshr_b32 s81, s78, 16
+; SI-NEXT: s_mov_b64 s[76:77], 0
; SI-NEXT: s_branch .LBB111_3
; SI-NEXT: .LBB111_2:
-; SI-NEXT: s_mov_b32 s6, s4
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $vcc_hi
-; SI-NEXT: ; implicit-def: $vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: s_mov_b32 s79, s5
+; SI-NEXT: s_mov_b32 vcc_lo, s13
+; SI-NEXT: s_mov_b32 s95, s15
+; SI-NEXT: s_mov_b32 s92, s41
+; SI-NEXT: s_mov_b32 s93, s40
+; SI-NEXT: s_mov_b32 s94, s12
+; SI-NEXT: s_mov_b32 s91, s14
+; SI-NEXT: s_mov_b32 s90, s42
+; SI-NEXT: s_mov_b32 s89, s43
+; SI-NEXT: s_mov_b32 s88, s44
+; SI-NEXT: s_mov_b64 s[76:77], -1
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr44
; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr81
; SI-NEXT: .LBB111_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_mov_b32 s5, s11
-; SI-NEXT: s_mov_b32 s11, s10
-; SI-NEXT: s_mov_b32 s10, s9
-; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_mov_b32 s9, s8
-; SI-NEXT: s_mov_b32 s8, s10
-; SI-NEXT: s_mov_b32 s10, s69
-; SI-NEXT: s_mov_b32 s69, s11
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s76, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s76, 1
; SI-NEXT: s_cbranch_scc1 .LBB111_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s65, s65, 3
-; SI-NEXT: s_and_b32 s4, s65, 0xff
-; SI-NEXT: s_lshl_b32 s5, s64, 8
-; SI-NEXT: s_add_i32 s54, s54, 3
+; SI-NEXT: v_readlane_b32 s4, v44, 11
+; SI-NEXT: s_add_i32 s4, s4, 3
+; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: s_lshl_b32 s5, s79, 8
+; SI-NEXT: s_add_i32 s6, s95, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s7, s54, 0xff
+; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s55, 24
-; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: s_add_i32 s53, s53, 3
+; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s53, 0xff
-; SI-NEXT: s_lshl_b32 s7, s52, 8
-; SI-NEXT: s_add_i32 s50, s50, 3
-; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: s_and_b32 s8, s50, 0xff
+; SI-NEXT: s_add_i32 s5, s92, 3
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: s_lshl_b32 s6, s93, 8
+; SI-NEXT: s_add_i32 s7, s91, 3
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s7, s7, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s7, s51, 24
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_lshl_b32 s6, s94, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_add_i32 s6, s66, 3
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s65, 8
+; SI-NEXT: s_add_i32 s8, s89, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s8, s8, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s90, 24
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_add_i32 s49, s49, 3
-; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: s_and_b32 s7, s49, 0xff
-; SI-NEXT: s_lshl_b32 s8, s48, 8
-; SI-NEXT: s_add_i32 s38, s38, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s7, s88, 3
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s8, s75, 8
+; SI-NEXT: s_add_i32 s9, s73, 3
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s9, s38, 0xff
+; SI-NEXT: s_and_b32 s9, s9, 0xff
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s8, s39, 24
+; SI-NEXT: s_lshl_b32 s8, s31, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_add_i32 s37, s37, 3
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s37, 0xff
-; SI-NEXT: s_lshl_b32 s9, s36, 8
-; SI-NEXT: s_add_i32 s34, s34, 3
+; SI-NEXT: s_add_i32 s8, s54, 3
+; SI-NEXT: s_and_b32 s8, s8, 0xff
+; SI-NEXT: s_lshl_b32 s9, s53, 8
+; SI-NEXT: s_add_i32 s10, s51, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s10, s34, 0xff
+; SI-NEXT: s_and_b32 s10, s10, 0xff
; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_lshl_b32 s9, s35, 24
+; SI-NEXT: s_lshl_b32 s9, s52, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s9, s9, s10
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_add_i32 s89, s89, 3
-; SI-NEXT: s_add_i32 s11, s8, 0x3000000
-; SI-NEXT: s_and_b32 s8, s89, 0xff
-; SI-NEXT: s_lshl_b32 s9, s31, 8
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_lshl_b32 s9, s30, 24
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s6, s9, s6
-; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_add_i32 s77, s77, 3
-; SI-NEXT: s_add_i32 s12, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s77, 0xff
-; SI-NEXT: s_lshl_b32 s8, s75, 8
-; SI-NEXT: s_add_i32 s72, s72, 3
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_and_b32 s9, s72, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s8, s63, 24
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_add_i32 s73, s73, 3
-; SI-NEXT: s_add_i32 s63, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s73, 0xff
-; SI-NEXT: s_lshl_b32 s8, s62, 8
-; SI-NEXT: s_add_i32 s60, s60, 3
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_and_b32 s9, s60, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s8, s59, 24
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_add_i32 s61, s61, 3
-; SI-NEXT: s_add_i32 s59, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s61, 0xff
-; SI-NEXT: s_lshl_b32 s8, s58, 8
-; SI-NEXT: s_add_i32 s56, s56, 3
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_and_b32 s9, s56, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s8, s47, 24
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_add_i32 s57, s57, 3
-; SI-NEXT: s_add_i32 s47, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s57, 0xff
-; SI-NEXT: s_lshl_b32 s8, s46, 8
-; SI-NEXT: s_add_i32 s44, s44, 3
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_and_b32 s9, s44, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s8, s43, 24
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_add_i32 s45, s45, 3
-; SI-NEXT: s_add_i32 s8, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s45, 0xff
-; SI-NEXT: s_lshl_b32 s9, s42, 8
-; SI-NEXT: s_add_i32 s40, s40, 3
-; SI-NEXT: s_or_b32 s6, s9, s6
-; SI-NEXT: s_and_b32 s10, s40, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s9, s15, 24
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_or_b32 s6, s9, s6
-; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: s_add_i32 s15, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s41, 0xff
-; SI-NEXT: s_lshl_b32 s9, s14, 8
-; SI-NEXT: s_or_b32 s6, s9, s6
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_add_i32 s87, s87, 3
-; SI-NEXT: s_and_b32 s9, s6, 0xffff
-; SI-NEXT: v_readlane_b32 s6, v44, 5
-; SI-NEXT: s_and_b32 s10, s87, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 24
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_or_b32 s6, s6, s10
-; SI-NEXT: s_or_b32 s6, s6, s9
-; SI-NEXT: s_add_i32 s9, s6, 0x3000000
+; SI-NEXT: s_add_i32 s9, s63, 3
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_lshl_b32 s10, s64, 8
+; SI-NEXT: s_add_i32 s82, s82, 3
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_and_b32 s11, s82, 0xff
+; SI-NEXT: s_addk_i32 s9, 0x300
+; SI-NEXT: s_lshl_b32 s10, s55, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_add_i32 s10, s50, 3
+; SI-NEXT: s_and_b32 s10, s10, 0xff
+; SI-NEXT: s_lshl_b32 s11, s49, 8
+; SI-NEXT: s_add_i32 s12, s39, 3
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_addk_i32 s10, 0x300
+; SI-NEXT: s_lshl_b32 s11, s48, 24
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_add_i32 s11, s38, 3
+; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: s_lshl_b32 s12, s37, 8
+; SI-NEXT: s_add_i32 s13, s61, 3
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s13, s13, 0xff
+; SI-NEXT: s_addk_i32 s11, 0x300
+; SI-NEXT: s_lshl_b32 s12, s34, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_or_b32 s12, s12, s13
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_add_i32 s12, s36, 3
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_lshl_b32 s13, s35, 8
+; SI-NEXT: s_add_i32 s14, s59, 3
+; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_addk_i32 s12, 0x300
+; SI-NEXT: s_lshl_b32 s13, s30, 24
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_or_b32 s13, s13, s14
+; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_add_i32 s13, s57, 3
+; SI-NEXT: s_and_b32 s13, s13, 0xff
+; SI-NEXT: s_lshl_b32 s14, s47, 8
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: s_addk_i32 s13, 0x300
+; SI-NEXT: s_and_b32 s14, s13, 0xffff
+; SI-NEXT: v_readlane_b32 s13, v44, 8
+; SI-NEXT: s_lshl_b32 s15, s13, 24
+; SI-NEXT: v_readlane_b32 s13, v44, 9
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s6, v44, 4
-; SI-NEXT: s_and_b32 s10, s13, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: s_or_b32 s6, s6, s10
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_and_b32 s10, s6, 0xffff
-; SI-NEXT: v_readlane_b32 s6, v44, 2
-; SI-NEXT: s_lshl_b32 s13, s6, 24
-; SI-NEXT: v_readlane_b32 s6, v44, 3
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_or_b32 s6, s13, s6
-; SI-NEXT: s_or_b32 s6, s6, s10
-; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_add_i32 s13, s6, 0x3000000
-; SI-NEXT: s_and_b32 s6, s28, 0xff
-; SI-NEXT: s_lshl_b32 s10, s29, 8
-; SI-NEXT: s_or_b32 s6, s10, s6
-; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_and_b32 s10, s6, 0xffff
-; SI-NEXT: v_readlane_b32 s6, v44, 1
-; SI-NEXT: s_lshl_b32 s14, s6, 24
-; SI-NEXT: v_readlane_b32 s6, v44, 0
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_or_b32 s6, s14, s6
+; SI-NEXT: s_and_b32 s13, s13, 0xff
+; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: s_or_b32 s13, s15, s13
+; SI-NEXT: s_or_b32 s13, s13, s14
+; SI-NEXT: v_readlane_b32 s14, v44, 10
+; SI-NEXT: s_add_i32 s14, s14, 3
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s15, s45, 8
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: s_addk_i32 s14, 0x300
+; SI-NEXT: s_and_b32 s15, s14, 0xffff
+; SI-NEXT: v_readlane_b32 s14, v44, 6
+; SI-NEXT: s_lshl_b32 s40, s14, 24
+; SI-NEXT: v_readlane_b32 s14, v44, 7
+; SI-NEXT: s_add_i32 s14, s14, 3
+; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_or_b32 s14, s40, s14
+; SI-NEXT: s_or_b32 s14, s14, s15
+; SI-NEXT: v_readlane_b32 s15, v44, 5
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: s_and_b32 s40, s15, 0xff
+; SI-NEXT: v_readlane_b32 s15, v44, 4
+; SI-NEXT: s_lshl_b32 s15, s15, 8
+; SI-NEXT: s_or_b32 s15, s15, s40
+; SI-NEXT: s_addk_i32 s15, 0x300
+; SI-NEXT: s_and_b32 s40, s15, 0xffff
+; SI-NEXT: v_readlane_b32 s15, v44, 2
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_or_b32 s6, s6, s10
-; SI-NEXT: s_and_b32 s10, s24, 0xff
-; SI-NEXT: s_lshl_b32 s14, s25, 8
+; SI-NEXT: s_lshl_b32 s41, s15, 24
+; SI-NEXT: v_readlane_b32 s15, v44, 3
+; SI-NEXT: s_and_b32 s24, s24, 0xff
+; SI-NEXT: s_lshl_b32 s25, s25, 8
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_or_b32 s10, s14, s10
-; SI-NEXT: s_and_b32 s24, s26, 0xff
-; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s14, s27, 24
-; SI-NEXT: s_lshl_b32 s24, s24, 16
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_or_b32 s14, s14, s24
-; SI-NEXT: s_or_b32 s10, s14, s10
-; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: s_or_b32 s24, s25, s24
+; SI-NEXT: s_and_b32 s26, s26, 0xff
+; SI-NEXT: s_and_b32 s15, s15, 0xff
+; SI-NEXT: s_addk_i32 s24, 0x300
+; SI-NEXT: s_lshl_b32 s25, s27, 24
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s15, s15, 16
+; SI-NEXT: s_and_b32 s24, s24, 0xffff
+; SI-NEXT: s_or_b32 s25, s25, s26
+; SI-NEXT: s_or_b32 s15, s41, s15
+; SI-NEXT: s_or_b32 s24, s25, s24
+; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s14, s10, 0x3000000
-; SI-NEXT: s_and_b32 s10, s20, 0xff
-; SI-NEXT: s_lshl_b32 s20, s21, 8
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_or_b32 s15, s15, s40
+; SI-NEXT: s_add_i32 s40, s24, 0x3000000
+; SI-NEXT: s_and_b32 s24, s28, 0xff
+; SI-NEXT: s_lshl_b32 s25, s29, 8
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s10, s20, s10
-; SI-NEXT: s_and_b32 s21, s22, 0xff
+; SI-NEXT: s_or_b32 s24, s25, s24
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s20, s23, 24
-; SI-NEXT: s_lshl_b32 s21, s21, 16
+; SI-NEXT: s_addk_i32 s24, 0x300
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s19, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_and_b32 s25, s24, 0xffff
+; SI-NEXT: v_readlane_b32 s24, v44, 0
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_lshl_b32 s26, s24, 24
+; SI-NEXT: v_readlane_b32 s24, v44, 1
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_add_i32 s42, s16, 0x3000000
+; SI-NEXT: s_and_b32 s16, s20, 0xff
+; SI-NEXT: s_lshl_b32 s17, s21, 8
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_and_b32 s24, s24, 0xff
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_and_b32 s18, s22, 0xff
+; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: s_addk_i32 s16, 0x300
+; SI-NEXT: s_lshl_b32 s17, s23, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_or_b32 s24, s26, s24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_or_b32 s10, s20, s10
+; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_add_i32 s7, s7, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
+; SI-NEXT: s_add_i32 s7, s7, 0x3000000
+; SI-NEXT: s_add_i32 s8, s8, 0x3000000
+; SI-NEXT: s_add_i32 s9, s9, 0x3000000
; SI-NEXT: s_add_i32 s10, s10, 0x3000000
-; SI-NEXT: s_add_i32 s16, s16, 0x3000000
-; SI-NEXT: s_and_b32 s76, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s16, 16
-; SI-NEXT: s_and_b32 s79, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s88, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s14, 16
-; SI-NEXT: s_and_b32 s91, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s70, s6, 16
-; SI-NEXT: s_and_b32 s92, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s13, 16
-; SI-NEXT: s_and_b32 s94, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s95, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s93, s15, 16
-; SI-NEXT: s_and_b32 vcc_hi, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_and_b32 s86, s47, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, s47, 16
-; SI-NEXT: s_and_b32 s85, s59, 0xffff0000
-; SI-NEXT: s_lshl_b32 s69, s59, 16
-; SI-NEXT: s_and_b32 s84, s63, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, s63, 16
-; SI-NEXT: s_and_b32 s82, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s12, 16
-; SI-NEXT: s_and_b32 s80, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s11, 16
-; SI-NEXT: s_and_b32 s68, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s7, 16
-; SI-NEXT: s_and_b32 s66, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s5, 16
-; SI-NEXT: s_and_b32 s5, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s4, 16
+; SI-NEXT: s_add_i32 s11, s11, 0x3000000
+; SI-NEXT: s_add_i32 s12, s12, 0x3000000
+; SI-NEXT: s_add_i32 s13, s13, 0x3000000
+; SI-NEXT: s_add_i32 s14, s14, 0x3000000
+; SI-NEXT: s_add_i32 s15, s15, 0x3000000
+; SI-NEXT: s_add_i32 s41, s24, 0x3000000
+; SI-NEXT: s_add_i32 s43, s16, 0x3000000
+; SI-NEXT: s_lshr_b64 s[44:45], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[56:57], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s71, s43, 16
+; SI-NEXT: s_lshr_b32 s70, s41, 16
+; SI-NEXT: s_lshr_b32 s68, s15, 16
+; SI-NEXT: s_lshr_b32 s69, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s11, 16
+; SI-NEXT: s_lshr_b32 s67, s9, 16
+; SI-NEXT: s_lshr_b32 s80, s7, 16
+; SI-NEXT: s_lshr_b32 s81, s5, 16
; SI-NEXT: .LBB111_5: ; %end
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s79
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s88
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s78
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s91
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s70
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s92
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s90
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s94
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s95
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s93
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, vcc_hi
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s86
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, vcc_lo
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s85
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s69
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s84
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s83
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s82
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s81
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s80
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s71
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s68
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s67
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s66
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s5
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s12
-; SI-NEXT: v_readlane_b32 s30, v43, 31
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_readlane_b32 s31, v43, 32
+; SI-NEXT: s_and_b32 s16, s42, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s44, 16
+; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_and_b32 s17, s43, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s71, 16
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_and_b32 s18, s40, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s46, 16
+; SI-NEXT: s_or_b32 s18, s18, s19
+; SI-NEXT: s_and_b32 s19, s41, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s70, 16
+; SI-NEXT: s_or_b32 s19, s19, s20
+; SI-NEXT: s_and_b32 s14, s14, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s56, 16
+; SI-NEXT: s_or_b32 s14, s14, s20
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s68, 16
+; SI-NEXT: s_or_b32 s15, s15, s20
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s58, 16
+; SI-NEXT: s_or_b32 s12, s12, s20
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s69, 16
+; SI-NEXT: s_or_b32 s13, s13, s20
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s60, 16
+; SI-NEXT: s_or_b32 s10, s10, s20
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s20, vcc_hi, 16
+; SI-NEXT: s_or_b32 s11, s11, s20
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s62, 16
+; SI-NEXT: s_or_b32 s8, s8, s20
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s67, 16
+; SI-NEXT: s_or_b32 s9, s9, s20
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s72, 16
+; SI-NEXT: s_or_b32 s6, s6, s20
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s80, 16
+; SI-NEXT: s_or_b32 s7, s7, s20
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s74, 16
+; SI-NEXT: s_or_b32 s4, s4, s20
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s81, 16
+; SI-NEXT: s_or_b32 s5, s5, s20
+; SI-NEXT: v_readlane_b32 s30, v43, 34
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s14
+; SI-NEXT: v_mov_b32_e32 v5, s15
+; SI-NEXT: v_mov_b32_e32 v6, s12
+; SI-NEXT: v_mov_b32_e32 v7, s13
+; SI-NEXT: v_mov_b32_e32 v8, s10
+; SI-NEXT: v_mov_b32_e32 v9, s11
+; SI-NEXT: v_mov_b32_e32 v10, s8
+; SI-NEXT: v_mov_b32_e32 v11, s9
+; SI-NEXT: v_mov_b32_e32 v12, s6
+; SI-NEXT: v_mov_b32_e32 v13, s7
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v15, s5
+; SI-NEXT: v_readlane_b32 s31, v43, 35
+; SI-NEXT: v_readlane_b32 s99, v43, 33
+; SI-NEXT: v_readlane_b32 s98, v43, 32
+; SI-NEXT: v_readlane_b32 s97, v43, 31
; SI-NEXT: v_readlane_b32 s96, v43, 30
; SI-NEXT: v_readlane_b32 s87, v43, 29
; SI-NEXT: v_readlane_b32 s86, v43, 28
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 009da205edba1..bc6ad6920e444 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2025,38 +2025,28 @@ define <4 x bfloat> @bitcast_i64_to_v4bf16(i64 %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_i64_to_v4bf16:
@@ -2126,39 +2116,33 @@ define inreg <4 x bfloat> @bitcast_i64_to_v4bf16_scalar(i64 inreg %a, i32 inreg
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s17, 16
-; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB21_3
; SI-NEXT: .LBB21_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB21_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB21_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_u32 s4, s16, 3
-; SI-NEXT: s_addc_u32 s5, s17, 0
-; SI-NEXT: s_and_b32 s7, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s5, 16
-; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s4, 16
+; SI-NEXT: s_add_u32 s16, s16, 3
+; SI-NEXT: s_addc_u32 s17, s17, 0
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: .LBB21_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_i64_to_v4bf16_scalar:
@@ -2249,15 +2233,13 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_i64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -2269,25 +2251,23 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB22_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_alignbit_b32 v0, v0, v5, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB22_2
; SI-NEXT: .LBB22_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -2514,45 +2494,45 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; SI-LABEL: bitcast_v4bf16_to_i64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
-; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB23_3
; SI-NEXT: .LBB23_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
; SI-NEXT: .LBB23_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
; SI-NEXT: s_cbranch_scc1 .LBB23_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB23_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4bf16_to_i64_scalar:
@@ -5451,36 +5431,27 @@ define <4 x bfloat> @bitcast_f64_to_v4bf16(double %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_f64_to_v4bf16:
@@ -5547,44 +5518,36 @@ define inreg <4 x bfloat> @bitcast_f64_to_v4bf16_scalar(double inreg %a, i32 inr
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s9, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s17, 16
-; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB45_3
; SI-NEXT: .LBB45_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: .LBB45_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB45_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: s_branch .LBB45_6
; SI-NEXT: .LBB45_5:
-; SI-NEXT: v_mov_b32_e32 v3, s9
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v3, s8
+; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: .LBB45_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_f64_to_v4bf16_scalar:
@@ -5674,15 +5637,13 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_f64:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -5694,25 +5655,23 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB46_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_alignbit_b32 v0, v0, v5, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB46_2
; SI-NEXT: .LBB46_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -5939,45 +5898,45 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; SI-LABEL: bitcast_v4bf16_to_f64_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
-; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB47_3
; SI-NEXT: .LBB47_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
; SI-NEXT: .LBB47_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
; SI-NEXT: s_cbranch_scc1 .LBB47_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB47_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4bf16_to_f64_scalar:
@@ -8529,38 +8488,28 @@ define <4 x bfloat> @bitcast_v2i32_to_v4bf16(<2 x i32> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2i32_to_v4bf16:
@@ -8629,39 +8578,33 @@ define inreg <4 x bfloat> @bitcast_v2i32_to_v4bf16_scalar(<2 x i32> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s17, 16
-; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB65_3
; SI-NEXT: .LBB65_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: .LBB65_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB65_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_and_b32 s7, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s17, 16
-; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s16, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
+; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: .LBB65_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2i32_to_v4bf16_scalar:
@@ -8752,15 +8695,13 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_v2i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -8772,25 +8713,23 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB66_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_alignbit_b32 v0, v0, v5, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB66_2
; SI-NEXT: .LBB66_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -9017,45 +8956,45 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v4bf16_to_v2i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
-; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB67_3
; SI-NEXT: .LBB67_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
; SI-NEXT: .LBB67_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
; SI-NEXT: s_cbranch_scc1 .LBB67_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB67_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB67_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4bf16_to_v2i32_scalar:
@@ -11242,38 +11181,28 @@ define <4 x bfloat> @bitcast_v2f32_to_v4bf16(<2 x float> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
+; SI-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2f32_to_v4bf16:
@@ -11341,45 +11270,37 @@ define inreg <4 x bfloat> @bitcast_v2f32_to_v4bf16_scalar(<2 x float> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB81_3
; SI-NEXT: .LBB81_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: .LBB81_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB81_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: s_branch .LBB81_6
; SI-NEXT: .LBB81_5:
-; SI-NEXT: v_mov_b32_e32 v0, s9
-; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: v_mov_b32_e32 v2, s7
-; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v3, s8
+; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: .LBB81_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v2f32_to_v4bf16_scalar:
@@ -11472,15 +11393,13 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_v2f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -11492,25 +11411,23 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB82_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_alignbit_b32 v0, v0, v5, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB82_2
; SI-NEXT: .LBB82_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -11737,45 +11654,45 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; SI-LABEL: bitcast_v4bf16_to_v2f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s8, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
-; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s8, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: s_branch .LBB83_3
; SI-NEXT: .LBB83_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
; SI-NEXT: .LBB83_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
; SI-NEXT: s_cbranch_scc1 .LBB83_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB83_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB83_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4bf16_to_v2f32_scalar:
@@ -13559,43 +13476,45 @@ define <4 x bfloat> @bitcast_v4i16_to_v4bf16(<4 x i16> %a, i32 %b) #0 {
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v4, v1, v7
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v5, v4, v6, 16
+; SI-NEXT: v_or_b32_e32 v2, v0, v6
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB92_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_or_b32_e32 v0, v2, v0
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x30000, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x30000, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v6, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 0x30000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v7, v0
+; SI-NEXT: v_add_i32_e32 v4, vcc, 0x30000, v0
+; SI-NEXT: v_alignbit_b32 v5, v4, v2, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
; SI-NEXT: .LBB92_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i16_to_v4bf16:
@@ -13666,52 +13585,52 @@ define inreg <4 x bfloat> @bitcast_v4i16_to_v4bf16_scalar(<4 x i16> inreg %a, i3
; SI-LABEL: bitcast_v4i16_to_v4bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: s_lshr_b32 s10, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s7, s16, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 16
-; SI-NEXT: s_lshl_b32 s6, s17, 16
-; SI-NEXT: s_lshl_b32 s8, s11, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s11, 16
+; SI-NEXT: s_or_b32 s13, s5, s6
+; SI-NEXT: s_or_b32 s4, s4, s12
+; SI-NEXT: s_lshr_b64 s[6:7], s[12:13], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s5, s13
; SI-NEXT: s_branch .LBB93_3
; SI-NEXT: .LBB93_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: .LBB93_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB93_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s10, 16
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: s_and_b32 s9, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s5, 16
-; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s10, s5, 16
; SI-NEXT: .LBB93_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4i16_to_v4bf16_scalar:
@@ -13811,55 +13730,35 @@ define <4 x i16> @bitcast_v4bf16_to_v4i16(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_v4i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v3
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB94_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
+; SI-NEXT: s_cbranch_execz .LBB94_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_alignbit_b32 v3, v1, v3, 16
-; SI-NEXT: .LBB94_4: ; %end
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_alignbit_b32 v1, v3, v1, 16
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; SI-NEXT: v_alignbit_b32 v4, v1, v2, 16
+; SI-NEXT: .LBB94_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -14076,55 +13975,48 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v4bf16_to_v4i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB95_3
; SI-NEXT: .LBB95_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: .LBB95_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB95_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; SI-NEXT: v_lshr_b64 v[1:2], v[4:5], 16
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v3, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_add_f32_e32 v2, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v3
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
-; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
-; SI-NEXT: .LBB95_5: ; %end
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
+; SI-NEXT: s_branch .LBB95_6
+; SI-NEXT: .LBB95_5:
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s7
+; SI-NEXT: .LBB95_6: ; %end
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -15456,54 +15348,54 @@ define <4 x bfloat> @bitcast_v4f16_to_v4bf16(<4 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4f16_to_v4bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v2, v1, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v5, v2, v1, 16
+; SI-NEXT: v_or_b32_e32 v4, v0, v1
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB100_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v6
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_or_b32_e32 v4, v0, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; SI-NEXT: v_or_b32_e32 v2, v1, v0
+; SI-NEXT: v_alignbit_b32 v5, v2, v5, 16
; SI-NEXT: .LBB100_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v2
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4f16_to_v4bf16:
@@ -15575,60 +15467,61 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; SI-LABEL: bitcast_v4f16_to_v4bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s7, s17, 16
-; SI-NEXT: s_lshr_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_lshl_b32 s9, s6, 16
-; SI-NEXT: s_lshl_b32 s10, s17, 16
-; SI-NEXT: s_lshl_b32 s11, s7, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: s_or_b32 s12, s5, s4
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB101_3
; SI-NEXT: .LBB101_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr5
; SI-NEXT: .LBB101_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB101_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s7
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s16
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s10
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s11
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_or_b32_e32 v0, v5, v0
; SI-NEXT: s_branch .LBB101_6
; SI-NEXT: .LBB101_5:
-; SI-NEXT: v_mov_b32_e32 v3, s11
-; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_mov_b32_e32 v1, s9
-; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v4, s10
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: .LBB101_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4f16_to_v4bf16_scalar:
@@ -15731,58 +15624,50 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_v4f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; SI-NEXT: v_alignbit_b32 v2, v1, v5, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
-; SI-NEXT: v_alignbit_b32 v3, v2, v0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v6
-; SI-NEXT: v_alignbit_b32 v0, v0, v7, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v4, v1, v6
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v5, v4, v7, 16
+; SI-NEXT: v_or_b32_e32 v2, v0, v7
+; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB102_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v7
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v2, v1, v2, 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT: v_alignbit_b32 v3, v2, v3, 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_alignbit_b32 v2, v2, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-NEXT: v_alignbit_b32 v4, v3, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
+; SI-NEXT: v_alignbit_b32 v5, v4, v0, 16
; SI-NEXT: .LBB102_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v2, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4bf16_to_v4f16:
@@ -16007,56 +15892,58 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; SI-LABEL: bitcast_v4bf16_to_v4f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v11
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v12
-; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[5:6], v[7:8], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: s_or_b32 s12, s5, s4
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB103_3
; SI-NEXT: .LBB103_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr5
; SI-NEXT: .LBB103_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB103_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v3, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
+; SI-NEXT: v_add_f32_e32 v2, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v3
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
-; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
-; SI-NEXT: .LBB103_5: ; %end
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
+; SI-NEXT: s_branch .LBB103_6
+; SI-NEXT: .LBB103_5:
+; SI-NEXT: v_mov_b32_e32 v3, s10
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v4, s12
+; SI-NEXT: v_mov_b32_e32 v5, s6
+; SI-NEXT: .LBB103_6: ; %end
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -17412,22 +17299,19 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v4bf16_to_v8i8:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v8, v1
+; SI-NEXT: v_mov_b32_e32 v9, v0
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v9
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v3
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -17439,31 +17323,29 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB108_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v11, 16
-; SI-NEXT: v_alignbit_b32 v4, v6, v9, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v11
+; SI-NEXT: v_or_b32_e32 v4, v1, v10
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v8
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB108_2
; SI-NEXT: .LBB108_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v11
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v8
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
; SI-NEXT: v_alignbit_b32 v4, v6, v1, 16
@@ -17798,64 +17680,70 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; SI-LABEL: bitcast_v4bf16_to_v8i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s17, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s16, 16
+; SI-NEXT: s_lshr_b32 s14, s17, 16
+; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v15
-; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; SI-NEXT: v_lshr_b64 v[12:13], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; SI-NEXT: v_lshr_b64 v[3:4], v[11:12], 24
-; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[11:12], 8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v12
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s15, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s14, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 8
+; SI-NEXT: s_lshr_b32 s7, s5, 8
+; SI-NEXT: s_bfe_u32 s9, s14, 0x80008
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_branch .LBB109_3
; SI-NEXT: .LBB109_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: .LBB109_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
; SI-NEXT: s_cbranch_scc1 .LBB109_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v15
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v5, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_lshr_b64 v[12:13], v[5:6], 16
+; SI-NEXT: v_lshr_b64 v[9:10], v[5:6], 16
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; SI-NEXT: v_lshr_b64 v[3:4], v[11:12], 24
-; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[11:12], 8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v12
-; SI-NEXT: .LBB109_5: ; %end
-; SI-NEXT: v_mov_b32_e32 v0, v11
-; SI-NEXT: v_mov_b32_e32 v2, v9
-; SI-NEXT: v_mov_b32_e32 v4, v12
-; SI-NEXT: v_mov_b32_e32 v5, v8
+; SI-NEXT: v_lshr_b64 v[3:4], v[8:9], 24
+; SI-NEXT: v_lshr_b64 v[4:5], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[8:9], 8
+; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; SI-NEXT: s_branch .LBB109_6
+; SI-NEXT: .LBB109_5:
+; SI-NEXT: v_mov_b32_e32 v7, s9
+; SI-NEXT: v_mov_b32_e32 v6, s14
+; SI-NEXT: v_mov_b32_e32 v9, s5
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: v_mov_b32_e32 v5, s7
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v1, s10
+; SI-NEXT: .LBB109_6: ; %end
+; SI-NEXT: v_mov_b32_e32 v0, v8
+; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_mov_b32_e32 v4, v9
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v4bf16_to_v8i8_scalar:
@@ -18225,79 +18113,78 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v3
; SI-NEXT: v_lshlrev_b32_e32 v11, 8, v5
-; SI-NEXT: v_lshlrev_b32_e32 v9, 24, v7
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v7
+; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v1
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v11
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v4, v7, v3
+; SI-NEXT: v_or_b32_e32 v3, v1, v4
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT: v_or_b32_e32 v3, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v5, v8, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT: v_or_b32_e32 v0, v0, v11
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v10, v9, v0
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v10
+; SI-NEXT: v_or_b32_e32 v1, v8, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v5, v3, v1, 16
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v4
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: .LBB110_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB110_4
; SI-NEXT: ; %bb.3: ; %cmp.true
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v4
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v6
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v2
-; SI-NEXT: v_or_b32_e32 v3, v11, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v0, v10, v0
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x300, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x300, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v4, v9, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v1, v8, v1
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_add_i32_e32 v4, vcc, 0x3000000, v3
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x3000000, v0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; SI-NEXT: v_add_i32_e32 v1, vcc, 0x3000000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v6
+; SI-NEXT: v_or_b32_e32 v0, v11, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 0x300, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v2, v7, v2
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, 0x3000000, v0
+; SI-NEXT: v_alignbit_b32 v5, v3, v1, 16
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; SI-NEXT: .LBB110_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v10
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i8_to_v4bf16:
@@ -18566,74 +18453,76 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_or_b32 s6, s5, s4
-; SI-NEXT: s_and_b32 s4, s18, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s19, 24
-; SI-NEXT: s_or_b32 s8, s5, s4
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s21, 8
+; SI-NEXT: s_lshl_b32 s5, s17, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s7, s4, 16
-; SI-NEXT: s_and_b32 s4, s22, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s23, 24
-; SI-NEXT: s_or_b32 s9, s5, s4
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s18, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s19, 24
+; SI-NEXT: s_or_b32 s10, s6, s5
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s6, s21, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s22, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s23, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s8, s7, s6
+; SI-NEXT: s_or_b32 s11, s5, s8
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 16
+; SI-NEXT: s_or_b32 s4, s4, s10
+; SI-NEXT: s_lshr_b32 s7, s8, 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s5, s11
; SI-NEXT: s_branch .LBB111_3
; SI-NEXT: .LBB111_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: .LBB111_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB111_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s21, 8
-; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s4, s16, 0xff
+; SI-NEXT: s_lshl_b32 s5, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s22, 0xff
+; SI-NEXT: s_and_b32 s6, s18, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s23, 24
+; SI-NEXT: s_lshl_b32 s5, s19, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s16, 0xff
-; SI-NEXT: s_lshl_b32 s6, s17, 8
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s6, s21, 8
+; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s7, s18, 0xff
+; SI-NEXT: s_and_b32 s7, s22, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s6, s19, 24
+; SI-NEXT: s_lshl_b32 s6, s23, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_and_b32 s8, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s5, 16
-; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s4, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s7, s5, 16
; SI-NEXT: .LBB111_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s7, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v8i8_to_v4bf16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index c20a4ae3cc604..baaa55b387caf 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -1660,50 +1660,35 @@ define <6 x bfloat> @bitcast_v3i32_to_v6bf16(<3 x i32> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
+; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v4
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3i32_to_v6bf16:
@@ -1775,50 +1760,41 @@ define inreg <6 x bfloat> @bitcast_v3i32_to_v6bf16_scalar(<3 x i32> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s19, 0
; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s7, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s18, 16
-; SI-NEXT: s_and_b32 s9, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s17, 16
-; SI-NEXT: s_and_b32 s11, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB9_3
; SI-NEXT: .LBB9_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: .LBB9_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB9_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_and_b32 s7, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s18, 16
-; SI-NEXT: s_and_b32 s9, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s17, 16
-; SI-NEXT: s_and_b32 s11, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s16, 16
+; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: .LBB9_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s7
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s10, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_and_b32 s7, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s7, s4
+; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3i32_to_v6bf16_scalar:
@@ -1915,19 +1891,16 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6bf16_to_v3i32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v6, v2
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v5, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -1939,35 +1912,32 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB10_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v8, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v6, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v0, v0, v8
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB10_2
; SI-NEXT: .LBB10_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v8
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v7
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -2262,57 +2232,56 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v6bf16_to_v3i32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s18, 16
-; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s7, s18, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v12
-; SI-NEXT: v_lshr_b64 v[0:1], v[8:9], 16
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v11
-; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s7, 16
+; SI-NEXT: s_or_b32 s6, s6, s8
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB11_3
; SI-NEXT: .LBB11_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
; SI-NEXT: .LBB11_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB11_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v3, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v3
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v3
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB11_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6bf16_to_v3i32_scalar:
@@ -4995,50 +4964,35 @@ define <6 x bfloat> @bitcast_v3f32_to_v6bf16(<3 x float> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
; SI-NEXT: v_add_f32_e32 v2, 1.0, v2
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
+; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
+; SI-NEXT: v_alignbit_b32 v4, v1, v0, 16
+; SI-NEXT: v_alignbit_b32 v3, v0, v2, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: ; %bb.4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v4
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3f32_to_v6bf16:
@@ -5109,58 +5063,46 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s19, 0
; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s18, 16
-; SI-NEXT: s_and_b32 s8, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s11, s16, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB25_3
; SI-NEXT: .LBB25_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: .LBB25_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB25_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
+; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[3:4], v[2:3], 16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; SI-NEXT: s_branch .LBB25_6
; SI-NEXT: .LBB25_5:
-; SI-NEXT: v_mov_b32_e32 v0, s11
-; SI-NEXT: v_mov_b32_e32 v1, s10
-; SI-NEXT: v_mov_b32_e32 v2, s9
-; SI-NEXT: v_mov_b32_e32 v5, s8
-; SI-NEXT: v_mov_b32_e32 v3, s7
-; SI-NEXT: v_mov_b32_e32 v4, s6
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v4, s10
+; SI-NEXT: v_mov_b32_e32 v3, s4
+; SI-NEXT: v_mov_b32_e32 v5, s6
; SI-NEXT: .LBB25_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[3:4], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v0, v0, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v3f32_to_v6bf16_scalar:
@@ -5261,19 +5203,16 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6bf16_to_v3f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v6, v2
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v5, v0
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v4, 1.0, v2
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -5285,35 +5224,32 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB26_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v8, 16
-; SI-NEXT: v_alignbit_b32 v1, v1, v6, 16
-; SI-NEXT: v_alignbit_b32 v2, v2, v4, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; SI-NEXT: v_or_b32_e32 v0, v0, v8
+; SI-NEXT: v_or_b32_e32 v1, v1, v7
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB26_2
; SI-NEXT: .LBB26_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v8
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v7
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -5608,57 +5544,56 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; SI-LABEL: bitcast_v6bf16_to_v3f32_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s18, 16
-; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s7, s18, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v12
-; SI-NEXT: v_lshr_b64 v[0:1], v[8:9], 16
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v11
-; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s11, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s7, 16
+; SI-NEXT: s_or_b32 s6, s6, s8
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB27_3
; SI-NEXT: .LBB27_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
; SI-NEXT: .LBB27_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB27_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v3, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v3
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v3
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_add_f32_e32 v2, s4, v3
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_add_f32_e32 v3, s4, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-NEXT: .LBB27_5:
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6bf16_to_v3f32_scalar:
@@ -7080,113 +7015,111 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v3
-; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v5
-; SI-NEXT: v_lshlrev_b32_e32 v14, 24, v7
-; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v11
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: v_lshlrev_b32_e32 v15, 8, v5
+; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v7
+; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v1
+; SI-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v11
+; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_cbranch_execz .LBB36_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v15
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_or_b32_e32 v4, v12, v3
+; SI-NEXT: v_or_b32_e32 v3, v1, v4
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v2
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT: v_or_b32_e32 v3, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v5, v12, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v17
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v11, v14, v0
+; SI-NEXT: v_or_b32_e32 v1, v16, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v7, v3, v1, 16
+; SI-NEXT: v_or_b32_e32 v1, v0, v1
; SI-NEXT: v_and_b32_e32 v0, 0xff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v9
-; SI-NEXT: v_or_b32_e32 v13, v1, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v15, v16, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v10
+; SI-NEXT: v_or_b32_e32 v0, v0, v9
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v2, v5, v2
+; SI-NEXT: v_or_b32_e32 v11, v0, v2
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: .LBB36_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB36_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v8
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v9
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_add_i32_e32 v5, vcc, 3, v10
-; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x300, v3
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v5, v16, v5
-; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: s_mov_b32 s7, 0x3000000
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_add_i32_e32 v8, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v3, vcc, 3, v4
; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT: v_add_i32_e32 v4, vcc, 3, v6
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v2
-; SI-NEXT: s_movk_i32 s6, 0x300
-; SI-NEXT: v_or_b32_e32 v3, v17, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_or_b32_e32 v0, v17, v0
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT: v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 0x300, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_or_b32_e32 v4, v14, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v1, v12, v1
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
+; SI-NEXT: v_or_b32_e32 v1, v16, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_add_i32_e32 v4, vcc, s7, v3
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x3000000, v0
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; SI-NEXT: s_mov_b32 s7, 0x3000000
+; SI-NEXT: v_add_i32_e32 v1, vcc, s7, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v4
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v6
+; SI-NEXT: s_movk_i32 s6, 0x300
+; SI-NEXT: v_or_b32_e32 v0, v15, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v2, v12, v2
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, 0x3000000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v8
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v10
+; SI-NEXT: v_or_b32_e32 v0, v9, v0
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 0x300, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v2, v5, v2
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_add_i32_e32 v11, vcc, 0x3000000, v0
+; SI-NEXT: v_alignbit_b32 v7, v3, v1, 16
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v11
; SI-NEXT: .LBB36_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v11
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v15
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v13
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v11
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v14
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v12i8_to_v6bf16:
@@ -7524,53 +7457,59 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_or_b32 s6, s5, s4
-; SI-NEXT: s_and_b32 s4, s18, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s19, 24
-; SI-NEXT: s_or_b32 s8, s5, s4
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s21, 8
+; SI-NEXT: s_lshl_b32 s5, s17, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_lshl_b32 s7, s4, 16
-; SI-NEXT: s_and_b32 s4, s22, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s23, 24
-; SI-NEXT: s_or_b32 s10, s5, s4
-; SI-NEXT: s_and_b32 s4, s24, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s25, 24
-; SI-NEXT: s_or_b32 s9, s5, s4
-; SI-NEXT: s_and_b32 s4, s26, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s27, 24
-; SI-NEXT: s_or_b32 s11, s5, s4
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s18, 0xff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s6, s19, 24
+; SI-NEXT: s_or_b32 s12, s6, s5
+; SI-NEXT: s_and_b32 s5, s24, 0xff
+; SI-NEXT: s_lshl_b32 s6, s25, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s26, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s27, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s8, s7, s6
+; SI-NEXT: s_or_b32 s10, s5, s8
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s6, s21, 8
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s22, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s23, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_or_b32 s9, s7, s6
+; SI-NEXT: s_or_b32 s13, s5, s9
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshr_b64 s[6:7], s[12:13], 16
+; SI-NEXT: s_or_b32 s4, s4, s12
+; SI-NEXT: s_lshr_b32 s7, s9, 16
+; SI-NEXT: s_lshr_b32 s11, s8, 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s5, s13
; SI-NEXT: s_branch .LBB37_3
; SI-NEXT: .LBB37_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: ; implicit-def: $sgpr11
; SI-NEXT: .LBB37_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cbranch_scc1 .LBB37_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_and_b32 s4, s24, 0xff
-; SI-NEXT: s_lshl_b32 s5, s25, 8
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s4, s16, 0xff
+; SI-NEXT: s_lshl_b32 s5, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s26, 0xff
+; SI-NEXT: s_and_b32 s6, s18, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s27, 24
+; SI-NEXT: s_lshl_b32 s5, s19, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -7586,41 +7525,38 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s16, 0xff
-; SI-NEXT: s_lshl_b32 s7, s17, 8
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_and_b32 s6, s24, 0xff
+; SI-NEXT: s_lshl_b32 s7, s25, 8
+; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s8, s18, 0xff
+; SI-NEXT: s_and_b32 s8, s26, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s19, 24
+; SI-NEXT: s_lshl_b32 s7, s27, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: s_and_b32 s8, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s10, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s5, 16
-; SI-NEXT: s_and_b32 s11, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s4, 16
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s10, s6, 0x3000000
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s7, s5, 16
+; SI-NEXT: s_lshr_b32 s11, s10, 16
; SI-NEXT: .LBB37_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s7, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s11, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v12i8_to_v6bf16_scalar:
@@ -7847,30 +7783,25 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6bf16_to_v12i8:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_mov_b32_e32 v12, v2
+; SI-NEXT: v_mov_b32_e32 v13, v1
+; SI-NEXT: v_mov_b32_e32 v14, v0
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v14
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: v_mul_f32_e32 v16, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v17, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v14, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v15, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v12, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v13, 1.0, v2
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v10
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -7882,43 +7813,40 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_setpc_b64 s[30:31]
; SI-NEXT: .LBB38_3: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v12
-; SI-NEXT: v_alignbit_b32 v0, v0, v17, 16
-; SI-NEXT: v_alignbit_b32 v4, v6, v15, 16
-; SI-NEXT: v_alignbit_b32 v8, v10, v13, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v13
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v0, v0, v17
+; SI-NEXT: v_or_b32_e32 v4, v1, v16
+; SI-NEXT: v_or_b32_e32 v8, v7, v15
; SI-NEXT: v_alignbit_b32 v3, v4, v0, 24
; SI-NEXT: v_alignbit_b32 v2, v4, v0, 16
; SI-NEXT: v_alignbit_b32 v1, v4, v0, 8
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v12
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; SI-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
+; SI-NEXT: v_bfe_u32 v11, v10, 8, 8
; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB38_2
; SI-NEXT: .LBB38_4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v17
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v17
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v14
; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v13
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
; SI-NEXT: v_alignbit_b32 v4, v6, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v12
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v15
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v11
; SI-NEXT: v_alignbit_b32 v8, v10, v1, 16
@@ -8368,85 +8296,91 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v6bf16_to_v12i8_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s18, 16
-; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s14, s18, 16
+; SI-NEXT: s_lshr_b32 s15, s17, 16
+; SI-NEXT: s_lshr_b32 s20, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v21
-; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v19
-; SI-NEXT: v_lshr_b64 v[17:18], v[5:6], 16
-; SI-NEXT: v_lshr_b64 v[13:14], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[16:17], 24
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v20
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v19
-; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v17
-; SI-NEXT: v_lshrrev_b32_e32 v12, 8, v13
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_lshr_b64 v[14:15], v[16:17], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[16:17], 8
-; SI-NEXT: s_branch .LBB39_3
-; SI-NEXT: .LBB39_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s20, 16
+; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s15, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 8
+; SI-NEXT: s_and_b32 s9, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s14, 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
+; SI-NEXT: s_or_b32 s11, s9, s11
+; SI-NEXT: s_lshr_b32 s7, s5, 8
+; SI-NEXT: s_lshr_b32 s9, s11, 8
+; SI-NEXT: s_bfe_u32 s19, s15, 0x80008
+; SI-NEXT: s_bfe_u32 s21, s14, 0x80008
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr19
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr21
; SI-NEXT: .LBB39_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
; SI-NEXT: s_cbranch_scc1 .LBB39_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v2, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: v_add_f32_e32 v1, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s17, 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
+; SI-NEXT: v_add_f32_e32 v5, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: v_lshr_b64 v[12:13], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s18, 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
-; SI-NEXT: v_lshr_b64 v[17:18], v[5:6], 16
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; SI-NEXT: v_lshr_b64 v[13:14], v[9:10], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[16:17], 24
-; SI-NEXT: v_lshr_b64 v[14:15], v[16:17], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[16:17], 8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v17
-; SI-NEXT: v_lshrrev_b32_e32 v12, 8, v13
+; SI-NEXT: v_add_f32_e32 v9, s4, v2
+; SI-NEXT: s_lshl_b32 s4, s14, 16
+; SI-NEXT: v_lshr_b64 v[13:14], v[5:6], 16
+; SI-NEXT: v_add_f32_e32 v11, s4, v2
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
+; SI-NEXT: v_lshr_b64 v[3:4], v[12:13], 24
+; SI-NEXT: v_lshr_b64 v[4:5], v[12:13], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[12:13], 8
+; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v13
+; SI-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v5
-; SI-NEXT: .LBB39_5: ; %end
-; SI-NEXT: v_mov_b32_e32 v0, v16
-; SI-NEXT: v_mov_b32_e32 v2, v14
-; SI-NEXT: v_mov_b32_e32 v4, v17
-; SI-NEXT: v_mov_b32_e32 v5, v8
-; SI-NEXT: v_mov_b32_e32 v8, v13
-; SI-NEXT: v_mov_b32_e32 v9, v12
+; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v11
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
+; SI-NEXT: v_mov_b32_e32 v11, s21
+; SI-NEXT: v_mov_b32_e32 v7, s19
+; SI-NEXT: v_mov_b32_e32 v10, s14
+; SI-NEXT: v_mov_b32_e32 v6, s15
+; SI-NEXT: v_mov_b32_e32 v8, s11
+; SI-NEXT: v_mov_b32_e32 v13, s5
+; SI-NEXT: v_mov_b32_e32 v12, s4
+; SI-NEXT: v_mov_b32_e32 v9, s9
+; SI-NEXT: v_mov_b32_e32 v5, s7
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v4, s8
+; SI-NEXT: v_mov_b32_e32 v1, s10
+; SI-NEXT: .LBB39_6: ; %end
+; SI-NEXT: v_mov_b32_e32 v0, v12
+; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_mov_b32_e32 v4, v13
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6bf16_to_v12i8_scalar:
@@ -11741,77 +11675,65 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6bf16_to_v6f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v8
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v4
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v2
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v4
; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; SI-NEXT: v_alignbit_b32 v4, v1, v7, 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
-; SI-NEXT: v_alignbit_b32 v5, v4, v0, 16
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; SI-NEXT: v_alignbit_b32 v0, v0, v11, 16
-; SI-NEXT: v_alignbit_b32 v3, v2, v9, 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v7, v1, v10
+; SI-NEXT: v_or_b32_e32 v3, v0, v11
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_alignbit_b32 v8, v7, v11, 16
+; SI-NEXT: v_or_b32_e32 v6, v0, v9
+; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr2
; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB48_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v11
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_alignbit_b32 v3, v2, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_alignbit_b32 v4, v1, v4, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; SI-NEXT: v_alignbit_b32 v3, v3, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v9
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_alignbit_b32 v6, v4, v0, 16
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v10
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_alignbit_b32 v7, v5, v0, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
+; SI-NEXT: v_alignbit_b32 v8, v7, v0, 16
; SI-NEXT: .LBB48_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: v_or_b32_e32 v1, v4, v1
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v8
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6bf16_to_v6f16:
@@ -12100,72 +12022,74 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; SI-LABEL: bitcast_v6bf16_to_v6f16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s18, 16
-; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s11, s18, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s12, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
-; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v16
-; SI-NEXT: v_lshr_b64 v[7:8], v[10:11], 16
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v15
-; SI-NEXT: v_lshr_b64 v[12:13], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[8:9], v[3:4], 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: s_or_b32 s13, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s11, 16
+; SI-NEXT: s_or_b32 s14, s5, s6
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB49_3
; SI-NEXT: .LBB49_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: .LBB49_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB49_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v10
-; SI-NEXT: v_lshr_b64 v[7:8], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
-; SI-NEXT: v_lshr_b64 v[8:9], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[12:13], v[0:1], 16
-; SI-NEXT: .LBB49_5: ; %end
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v7
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v12
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v7, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
+; SI-NEXT: v_add_f32_e32 v2, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v7
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_lshr_b64 v[5:6], v[2:3], 16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
+; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
+; SI-NEXT: v_mov_b32_e32 v7, s10
+; SI-NEXT: v_mov_b32_e32 v3, s11
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v5, s14
+; SI-NEXT: v_mov_b32_e32 v4, s13
+; SI-NEXT: v_mov_b32_e32 v8, s6
+; SI-NEXT: .LBB49_6: ; %end
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v8
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -12521,73 +12445,72 @@ define <6 x bfloat> @bitcast_v6f16_to_v6bf16(<6 x half> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6f16_to_v6bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT: s_cbranch_execz .LBB50_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v9
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v10
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v11
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_or_b32_e32 v6, v1, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v9
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_alignbit_b32 v7, v6, v1, 16
+; SI-NEXT: v_or_b32_e32 v3, v0, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v8, v0, v1
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; %bb.2: ; %Flow
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: .LBB50_2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB50_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v9
-; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v8, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v10
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v9
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; SI-NEXT: v_or_b32_e32 v3, v0, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT: v_or_b32_e32 v8, v2, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; SI-NEXT: v_or_b32_e32 v6, v0, v1
+; SI-NEXT: v_alignbit_b32 v7, v6, v7, 16
; SI-NEXT: .LBB50_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v4
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v8
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6f16_to_v6bf16:
@@ -12665,79 +12588,79 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; SI-LABEL: bitcast_v6f16_to_v6bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s7, s17, 16
-; SI-NEXT: s_lshr_b32 s6, s16, 16
+; SI-NEXT: s_lshr_b32 s11, s18, 16
+; SI-NEXT: s_lshr_b32 s10, s17, 16
+; SI-NEXT: s_lshr_b32 s12, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s9, s16, 16
-; SI-NEXT: s_lshl_b32 s10, s6, 16
-; SI-NEXT: s_lshl_b32 s11, s17, 16
-; SI-NEXT: s_lshl_b32 s12, s7, 16
-; SI-NEXT: s_lshl_b32 s13, s18, 16
-; SI-NEXT: s_lshl_b32 s14, s8, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: s_or_b32 s13, s5, s4
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s11, 16
+; SI-NEXT: s_or_b32 s14, s5, s6
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_branch .LBB51_3
; SI-NEXT: .LBB51_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr5
; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: .LBB51_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB51_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s8
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s18
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s12
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s11
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s7
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v2
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_or_b32_e32 v5, v1, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s10
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s17
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v6, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v7, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v8, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
+; SI-NEXT: v_or_b32_e32 v3, v8, v7
; SI-NEXT: s_branch .LBB51_6
; SI-NEXT: .LBB51_5:
-; SI-NEXT: v_mov_b32_e32 v4, s14
-; SI-NEXT: v_mov_b32_e32 v3, s13
-; SI-NEXT: v_mov_b32_e32 v5, s12
-; SI-NEXT: v_mov_b32_e32 v2, s11
-; SI-NEXT: v_mov_b32_e32 v1, s10
-; SI-NEXT: v_mov_b32_e32 v0, s9
+; SI-NEXT: v_mov_b32_e32 v4, s11
+; SI-NEXT: v_mov_b32_e32 v6, s10
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v3, s14
+; SI-NEXT: v_mov_b32_e32 v5, s13
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: .LBB51_6: ; %end
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: v_mul_f32_e32 v5, 1.0, v2
-; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; SI-NEXT: v_lshr_b64 v[2:3], v[3:4], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6f16_to_v6bf16_scalar:
@@ -12852,74 +12775,45 @@ define <6 x i16> @bitcast_v6bf16_to_v6i16(<6 x bfloat> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6bf16_to_v6i16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: v_mul_f32_e32 v11, 1.0, v0
-; SI-NEXT: v_mul_f32_e32 v10, 1.0, v6
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v1
-; SI-NEXT: v_mul_f32_e32 v6, 1.0, v5
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v8, 1.0, v4
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v8
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT: s_cbranch_execz .LBB52_4
-; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
-; SI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
-; SI-NEXT: v_alignbit_b32 v2, v3, v1, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; SI-NEXT: s_cbranch_execz .LBB52_2
+; SI-NEXT: ; %bb.1: ; %cmp.true
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_alignbit_b32 v1, v4, v1, 16
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT: v_alignbit_b32 v5, v1, v5, 16
-; SI-NEXT: .LBB52_4: ; %end
+; SI-NEXT: v_alignbit_b32 v1, v5, v1, 16
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; SI-NEXT: v_alignbit_b32 v0, v6, v0, 16
+; SI-NEXT: v_alignbit_b32 v2, v4, v2, 16
+; SI-NEXT: v_alignbit_b32 v6, v1, v3, 16
+; SI-NEXT: .LBB52_2: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT: v_or_b32_e32 v0, v0, v3
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v3
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v5
-; SI-NEXT: v_or_b32_e32 v1, v1, v4
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -13202,72 +13096,60 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; SI-LABEL: bitcast_v6bf16_to_v6i16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s18, 16
-; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s7, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v7
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB53_3
; SI-NEXT: .LBB53_2:
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: .LBB53_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB53_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v9
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_lshr_b64 v[6:7], v[4:5], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_lshr_b64 v[1:2], v[7:8], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
-; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
-; SI-NEXT: .LBB53_5: ; %end
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
+; SI-NEXT: s_lshl_b32 s4, s16, 16
+; SI-NEXT: v_mov_b32_e32 v7, 0x40c00000
+; SI-NEXT: v_add_f32_e32 v0, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: v_add_f32_e32 v8, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
+; SI-NEXT: v_add_f32_e32 v2, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[0:1], 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v7
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_lshr_b64 v[5:6], v[2:3], 16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_add_f32_e32 v6, s4, v7
+; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: v_add_f32_e32 v0, s4, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
+; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
+; SI-NEXT: v_mov_b32_e32 v7, s6
+; SI-NEXT: v_mov_b32_e32 v3, s7
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v5, s18
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v8, s8
+; SI-NEXT: .LBB53_6: ; %end
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v8
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -13606,61 +13488,63 @@ define <6 x bfloat> @bitcast_v6i16_to_v6bf16(<6 x i16> %a, i32 %b) #0 {
; SI-LABEL: bitcast_v6i16_to_v6bf16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr6
; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v6, v1, v11
+; SI-NEXT: v_or_b32_e32 v3, v0, v10
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; SI-NEXT: v_alignbit_b32 v7, v6, v10, 16
+; SI-NEXT: v_or_b32_e32 v8, v0, v9
; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; %bb.2: ; %Flow
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB54_4
; SI-NEXT: ; %bb.3: ; %cmp.true
-; SI-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v2, v5, v2
-; SI-NEXT: v_or_b32_e32 v1, v4, v1
-; SI-NEXT: v_or_b32_e32 v0, v3, v0
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0x30000, v2
-; SI-NEXT: v_add_i32_e32 v1, vcc, 0x30000, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x30000, v0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; SI-NEXT: v_or_b32_e32 v0, v10, v0
+; SI-NEXT: v_add_i32_e32 v3, vcc, 0x30000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v1
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v11, v0
+; SI-NEXT: v_add_i32_e32 v6, vcc, 0x30000, v0
+; SI-NEXT: v_add_i32_e32 v0, vcc, 3, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v9, v0
+; SI-NEXT: v_add_i32_e32 v8, vcc, 0x30000, v0
+; SI-NEXT: v_alignbit_b32 v7, v6, v3, 16
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v8
; SI-NEXT: .LBB54_4: ; %end
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v6
-; SI-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; SI-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v7
-; SI-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_mul_f32_e32 v3, 1.0, v8
-; SI-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6i16_to_v6bf16:
@@ -13737,68 +13621,67 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; SI-LABEL: bitcast_v6i16_to_v6bf16_scalar:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s14, s18, 16
-; SI-NEXT: s_lshr_b32 s13, s17, 16
-; SI-NEXT: s_lshr_b32 s12, s16, 16
+; SI-NEXT: s_lshr_b32 s10, s18, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
+; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_lshl_b32 s11, s12, 16
-; SI-NEXT: s_lshl_b32 s7, s17, 16
-; SI-NEXT: s_lshl_b32 s10, s13, 16
-; SI-NEXT: s_lshl_b32 s6, s18, 16
-; SI-NEXT: s_lshl_b32 s9, s14, 16
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_and_b32 s5, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s10, 16
+; SI-NEXT: s_or_b32 s12, s5, s6
+; SI-NEXT: s_and_b32 s5, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s11, 16
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s13, 16
+; SI-NEXT: s_or_b32 s15, s5, s6
+; SI-NEXT: s_or_b32 s4, s4, s14
+; SI-NEXT: s_lshr_b64 s[6:7], s[14:15], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s5, s15
; SI-NEXT: s_branch .LBB55_3
; SI-NEXT: .LBB55_2:
-; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: .LBB55_3: ; %Flow
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
; SI-NEXT: s_cbranch_scc1 .LBB55_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s13, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s13, 16
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_lshl_b32 s6, s11, 16
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s12, 16
+; SI-NEXT: s_and_b32 s6, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: s_add_i32 s6, s6, 0x30000
-; SI-NEXT: s_and_b32 s11, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s6, 16
-; SI-NEXT: s_and_b32 s10, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s7, s5, 16
-; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s4, 16
+; SI-NEXT: s_add_i32 s12, s6, 0x30000
+; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s11, s5, 16
+; SI-NEXT: s_lshr_b32 s10, s12, 16
; SI-NEXT: .LBB55_5: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s10
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s11, 16
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s6, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s10, 16
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v6i16_to_v6bf16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index dc1801f9b549f..c0346b9599210 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15023,33 +15023,31 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, bfloat %val) {
; GFX7LESS-LABEL: uniform_fadd_bf16:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
-; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_lshl_b32 s6, s6, 16
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
-; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
-; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; GFX7LESS-NEXT: s_not_b32 s2, s2
+; GFX7LESS-NEXT: s_and_b32 s7, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s10, s7, 3
+; GFX7LESS-NEXT: s_lshl_b32 s2, s6, 16
+; GFX7LESS-NEXT: s_lshl_b32 s6, 0xffff, s10
+; GFX7LESS-NEXT: s_not_b32 s11, s6
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
-; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s2, v0
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s11, v1
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v0
; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
@@ -15989,53 +15987,45 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX7LESS-NEXT: s_load_dword s4, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
-; GFX7LESS-NEXT: s_and_b32 s6, s4, 0xffff0000
-; GFX7LESS-NEXT: s_lshl_b32 s4, s4, 16
-; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s4
-; GFX7LESS-NEXT: v_mul_f32_e64 v1, 1.0, s6
+; GFX7LESS-NEXT: s_lshr_b32 s11, s4, 16
+; GFX7LESS-NEXT: s_lshl_b32 s10, s4, 16
+; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 16
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_and_b32 s4, s5, 0xffff0000
-; GFX7LESS-NEXT: s_lshl_b32 s5, s5, 16
-; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s5
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
-; GFX7LESS-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_lshr_b32 s4, s5, 16
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s4
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7LESS-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7LESS-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7LESS-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7LESS-NEXT: v_mov_b32_e32 v5, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v2
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7LESS-NEXT: v_add_f32_e32 v4, s11, v1
+; GFX7LESS-NEXT: v_add_f32_e32 v3, s10, v2
+; GFX7LESS-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX7LESS-NEXT: v_alignbit_b32 v1, v0, v3, 16
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7LESS-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7LESS-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB21_1
; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7LESS-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7LESS-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index e8ed1993b8978..32d1bffba90bb 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -5293,13 +5293,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: v_mov_b32_e32 v4, s4
; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_lshl_b32 s6, s4, 3
; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -5333,13 +5331,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: v_mov_b32_e32 v4, s4
; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s6, s4, 3
; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -5692,13 +5688,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: v_mov_b32_e32 v2, s4
; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_lshl_b32 s6, s4, 3
; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -5731,13 +5725,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: v_mov_b32_e32 v2, s4
; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s6, s4, 3
; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -6315,12 +6307,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
-; GFX7-NEXT: v_and_b32_e32 v9, -4, v4
+; GFX7-NEXT: v_and_b32_e32 v8, -4, v4
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 3, v4
-; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX7-NEXT: v_not_b32_e32 v10, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v4
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
+; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -6331,28 +6322,27 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v5, v9, s[8:11], 0 offen
+; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB18_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, v8, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v11
+; GFX7-NEXT: v_add_f32_e32 v4, v4, v10
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, v8, v4
-; GFX7-NEXT: v_and_b32_e32 v6, v5, v10
-; GFX7-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX7-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v5
+; GFX7-NEXT: v_mov_b32_e32 v5, v6
; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -6364,33 +6354,32 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB18_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v5, v6
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB18_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v8, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
-; GFX6-NEXT: v_and_b32_e32 v9, -4, v4
+; GFX6-NEXT: v_and_b32_e32 v8, -4, v4
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, 3, v4
-; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v8
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX6-NEXT: v_not_b32_e32 v10, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v4
+; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
+; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
; GFX6-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
@@ -6401,29 +6390,27 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v5, v9, s[8:11], 0 offen
+; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
; GFX6-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v8, v5
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v11
+; GFX6-NEXT: v_add_f32_e32 v4, v4, v10
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, v8, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, v5, v10
-; GFX6-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX6-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v5
+; GFX6-NEXT: v_mov_b32_e32 v5, v6
; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
@@ -6435,21 +6422,21 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX6-NEXT: v_mov_b32_e32 v5, v6
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_cbranch_execnz .LBB18_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, v8, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
@@ -9264,103 +9251,82 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -9733,43 +9699,34 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9779,49 +9736,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -10522,8 +10469,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -10534,7 +10480,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB28_1
; GFX7-NEXT: ; %bb.2:
@@ -10549,35 +10495,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_ushort v8, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v10, v4, s[8:11], 0 offen offset:1026
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB28_3
; GFX7-NEXT: ; %bb.4:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v6
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v9, 16, v7
; GFX7-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v7
-; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v8
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: v_add_f32_e32 v8, v8, v10
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v5, v8, v5, 16
-; GFX7-NEXT: v_alignbit_b32 v6, v7, v6, 16
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX7-NEXT: v_add_f32_e32 v10, v8, v9
+; GFX7-NEXT: v_add_f32_e32 v7, v7, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-NEXT: v_alignbit_b32 v7, v10, v7, 16
+; GFX7-NEXT: v_or_b32_e32 v8, v6, v8
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mov_b32_e32 v8, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
+; GFX7-NEXT: v_mov_b32_e32 v6, v7
+; GFX7-NEXT: v_mov_b32_e32 v7, v8
; GFX7-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -10589,32 +10529,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB28_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB28_5
; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
; GFX6-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
@@ -10626,7 +10563,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB28_1
; GFX6-NEXT: ; %bb.2:
@@ -10641,33 +10578,28 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_ushort v9, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v11, v4, s[8:11], 0 offen offset:1026
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB28_3
; GFX6-NEXT: ; %bb.4:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v9
-; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v10, 16, v7
; GFX6-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
; GFX6-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v7
-; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v5, v6, v5, 16
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_add_f32_e32 v7, v5, v10
+; GFX6-NEXT: v_add_f32_e32 v4, v4, v9
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX6-NEXT: v_alignbit_b32 v4, v7, v4, 16
+; GFX6-NEXT: v_or_b32_e32 v5, v6, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: v_mov_b32_e32 v6, v4
@@ -10689,19 +10621,17 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_cbranch_execnz .LBB28_5
; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v11
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -11080,103 +11010,82 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB29_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB29_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -11549,43 +11458,34 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11595,49 +11495,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB30_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -12016,103 +11906,82 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB31_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
@@ -12485,43 +12354,34 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12531,49 +12391,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB32_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
@@ -12946,43 +12796,34 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_add_f32_e32 v7, v2, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12992,49 +12833,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v2, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB33_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 6859d7fbbb95d..601252774378a 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4398,13 +4398,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: v_mov_b32_e32 v4, s4
; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_lshl_b32 s6, s4, 3
; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v0
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4439,13 +4438,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: v_mov_b32_e32 v4, s4
; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s6, s4, 3
; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -4799,13 +4797,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: v_mov_b32_e32 v2, s4
; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_lshl_b32 s6, s4, 3
; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4839,13 +4836,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: v_mov_b32_e32 v2, s4
; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s6, s4, 3
; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -5424,12 +5420,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
-; GFX7-NEXT: v_and_b32_e32 v9, -4, v4
+; GFX7-NEXT: v_and_b32_e32 v8, -4, v4
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 3, v4
-; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX7-NEXT: v_not_b32_e32 v10, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v4
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
+; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -5440,29 +5435,29 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v5, v9, s[8:11], 0 offen
+; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v4
; GFX7-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB15_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, v8, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_max_f32_e32 v4, v4, v11
+; GFX7-NEXT: v_max_f32_e32 v4, v4, v10
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, v8, v4
-; GFX7-NEXT: v_and_b32_e32 v6, v5, v10
-; GFX7-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX7-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v5
+; GFX7-NEXT: v_mov_b32_e32 v5, v6
; GFX7-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -5474,33 +5469,32 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v5, v6
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB15_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v8, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
-; GFX6-NEXT: v_and_b32_e32 v9, -4, v4
+; GFX6-NEXT: v_and_b32_e32 v8, -4, v4
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, 3, v4
-; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v8
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX6-NEXT: v_not_b32_e32 v10, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v4
+; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
+; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
; GFX6-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
@@ -5511,30 +5505,29 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v5, v9, s[8:11], 0 offen
+; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v10, 1.0, v4
; GFX6-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
; GFX6-NEXT: ; Child Loop BB15_4 Depth 2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v8, v5
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_max_f32_e32 v4, v4, v11
+; GFX6-NEXT: v_max_f32_e32 v4, v4, v10
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, v8, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, v5, v10
-; GFX6-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX6-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v5
+; GFX6-NEXT: v_mov_b32_e32 v5, v6
; GFX6-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
@@ -5546,21 +5539,21 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX6-NEXT: v_mov_b32_e32 v5, v6
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_cbranch_execnz .LBB15_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, v8, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
@@ -7757,103 +7750,90 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v1
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_max_f32_e32 v1, v7, v3
+; GFX7-NEXT: v_max_f32_e32 v2, v5, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
-; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_max_f32_e32 v1, v7, v3
+; GFX6-NEXT: v_max_f32_e32 v2, v5, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB19_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -8324,43 +8304,38 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v1
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_max_f32_e32 v1, v7, v3
+; GFX7-NEXT: v_max_f32_e32 v2, v5, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB20_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8370,49 +8345,43 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
-; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_max_f32_e32 v1, v7, v3
+; GFX6-NEXT: v_max_f32_e32 v2, v5, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB20_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%unused = atomicrmw fmax ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -9293,8 +9262,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -9305,7 +9273,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
@@ -9320,35 +9288,33 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_ushort v8, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v10, v4, s[8:11], 0 offen offset:1026
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_3
; GFX7-NEXT: ; %bb.4:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v8
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v9
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v8, v8, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v8, 16
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v7, 16
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX7-NEXT: v_max_f32_e32 v7, v7, v9
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_max_f32_e32 v10, v10, v5
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-NEXT: v_alignbit_b32 v7, v7, v10, 16
+; GFX7-NEXT: v_or_b32_e32 v8, v6, v8
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mov_b32_e32 v8, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
+; GFX7-NEXT: v_mov_b32_e32 v6, v7
+; GFX7-NEXT: v_mov_b32_e32 v7, v8
; GFX7-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -9360,32 +9326,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB21_5
; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
; GFX6-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
@@ -9397,7 +9360,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
@@ -9412,34 +9375,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_ushort v9, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v11, v4, s[8:11], 0 offen offset:1026
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_3
; GFX6-NEXT: ; %bb.4:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v11, 16, v9
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_mul_f32_e32 v9, 1.0, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX6-NEXT: v_mul_f32_e32 v10, 1.0, v4
; GFX6-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
; GFX6-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v11
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v5
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v9
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX6-NEXT: v_max_f32_e32 v4, v4, v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v6, v4, 16
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v7, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX6-NEXT: v_max_f32_e32 v4, v4, v9
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_max_f32_e32 v7, v7, v10
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX6-NEXT: v_or_b32_e32 v5, v6, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: v_mov_b32_e32 v6, v4
@@ -9461,18 +9422,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_cbranch_execnz .LBB21_5
; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v11
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index bd86786c1df67..50a7354fd2485 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4398,13 +4398,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: v_mov_b32_e32 v4, s4
; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_lshl_b32 s6, s4, 3
; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v0
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4439,13 +4438,12 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: v_mov_b32_e32 v4, s4
; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s6, s4, 3
; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -4799,13 +4797,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: v_mov_b32_e32 v2, s4
; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_lshl_b32 s6, s4, 3
; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4839,13 +4836,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: v_mov_b32_e32 v2, s4
; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s6, s4, 3
; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -5424,12 +5420,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
-; GFX7-NEXT: v_and_b32_e32 v9, -4, v4
+; GFX7-NEXT: v_and_b32_e32 v8, -4, v4
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 3, v4
-; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX7-NEXT: v_not_b32_e32 v10, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v4
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
+; GFX7-NEXT: v_not_b32_e32 v9, v4
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -5440,29 +5435,29 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_dword v5, v9, s[8:11], 0 offen
+; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_1
; GFX7-NEXT: ; %bb.2:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v4
; GFX7-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB15_4 Depth 2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, v8, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_min_f32_e32 v4, v4, v11
+; GFX7-NEXT: v_min_f32_e32 v4, v4, v10
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, v8, v4
-; GFX7-NEXT: v_and_b32_e32 v6, v5, v10
-; GFX7-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX7-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v5
+; GFX7-NEXT: v_mov_b32_e32 v5, v6
; GFX7-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -5474,33 +5469,32 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB15_4
; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v5, v6
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB15_3
; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v8, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
-; GFX6-NEXT: v_and_b32_e32 v9, -4, v4
+; GFX6-NEXT: v_and_b32_e32 v8, -4, v4
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, 3, v4
-; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v8
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX6-NEXT: v_not_b32_e32 v10, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v4
+; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
+; GFX6-NEXT: v_not_b32_e32 v9, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
; GFX6-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
@@ -5511,30 +5505,29 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_dword v5, v9, s[8:11], 0 offen
+; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_1
; GFX6-NEXT: ; %bb.2:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v10, 1.0, v4
; GFX6-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
; GFX6-NEXT: ; Child Loop BB15_4 Depth 2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v8, v5
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_min_f32_e32 v4, v4, v11
+; GFX6-NEXT: v_min_f32_e32 v4, v4, v10
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, v8, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, v5, v10
-; GFX6-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX6-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
; GFX6-NEXT: s_mov_b64 s[12:13], exec
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v5
+; GFX6-NEXT: v_mov_b32_e32 v5, v6
; GFX6-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
; GFX6-NEXT: v_readfirstlane_b32 s8, v0
@@ -5546,21 +5539,21 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB15_4
; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX6-NEXT: v_mov_b32_e32 v5, v6
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_cbranch_execnz .LBB15_3
; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, v8, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
@@ -7757,103 +7750,90 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v1
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_min_f32_e32 v1, v7, v3
+; GFX7-NEXT: v_min_f32_e32 v2, v5, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
-; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_min_f32_e32 v1, v7, v3
+; GFX6-NEXT: v_min_f32_e32 v2, v5, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB19_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -8324,43 +8304,38 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: v_mov_b32_e32 v1, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v0
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v1
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_min_f32_e32 v1, v7, v3
+; GFX7-NEXT: v_min_f32_e32 v2, v5, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB20_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8370,49 +8345,43 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s20
-; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_ushort v1, v2, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
-; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v0
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_min_f32_e32 v1, v7, v3
+; GFX6-NEXT: v_min_f32_e32 v2, v5, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_alignbit_b32 v5, v1, v2, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB20_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
%unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -9293,8 +9262,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -9305,7 +9273,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
+; GFX7-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_1
; GFX7-NEXT: ; %bb.2:
@@ -9320,35 +9288,33 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX7-NEXT: buffer_load_ushort v8, v4, s[8:11], 0 offen offset:1026
+; GFX7-NEXT: buffer_load_ushort v10, v4, s[8:11], 0 offen offset:1026
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_3
; GFX7-NEXT: ; %bb.4:
; GFX7-NEXT: s_mov_b64 exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_mov_b64 s[6:7], 0
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Loop Header: Depth=1
; GFX7-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v8
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v9
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v8, v8, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v8, 16
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v7, 16
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX7-NEXT: v_min_f32_e32 v7, v7, v9
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_min_f32_e32 v10, v10, v5
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-NEXT: v_alignbit_b32 v7, v7, v10, 16
+; GFX7-NEXT: v_or_b32_e32 v8, v6, v8
; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mov_b32_e32 v8, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
+; GFX7-NEXT: v_mov_b32_e32 v6, v7
+; GFX7-NEXT: v_mov_b32_e32 v7, v8
; GFX7-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
; GFX7-NEXT: v_readfirstlane_b32 s8, v0
@@ -9360,32 +9326,29 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[8:11], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB21_6
; GFX7-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX7-NEXT: s_mov_b64 exec, s[12:13]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB21_5
; GFX7-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v7
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
; GFX6-NEXT: s_mov_b64 s[6:7], exec
; GFX6-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
@@ -9397,7 +9360,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_ushort v7, v4, s[8:11], 0 offen offset:1024
+; GFX6-NEXT: buffer_load_ushort v6, v4, s[8:11], 0 offen offset:1024
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_1
; GFX6-NEXT: ; %bb.2:
@@ -9412,34 +9375,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX6-NEXT: buffer_load_ushort v9, v4, s[8:11], 0 offen offset:1026
+; GFX6-NEXT: buffer_load_ushort v11, v4, s[8:11], 0 offen offset:1026
; GFX6-NEXT: ; implicit-def: $vgpr4
; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB21_3
; GFX6-NEXT: ; %bb.4:
; GFX6-NEXT: s_mov_b64 exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v11, 16, v9
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_mul_f32_e32 v9, 1.0, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_mov_b64 s[6:7], 0
-; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX6-NEXT: v_mul_f32_e32 v10, 1.0, v4
; GFX6-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Loop Header: Depth=1
; GFX6-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v11
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v5
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v9
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX6-NEXT: v_min_f32_e32 v4, v4, v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v6, v4, 16
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v7, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX6-NEXT: v_min_f32_e32 v4, v4, v9
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_min_f32_e32 v7, v7, v10
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX6-NEXT: v_or_b32_e32 v5, v6, v5
; GFX6-NEXT: s_mov_b64 s[12:13], exec
; GFX6-NEXT: v_mov_b32_e32 v7, v5
; GFX6-NEXT: v_mov_b32_e32 v6, v4
@@ -9461,18 +9422,16 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX6-NEXT: s_mov_b64 exec, s[12:13]
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_cbranch_execnz .LBB21_5
; GFX6-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v11
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 9d8929bb26b7b..b616fd9a9ea4c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -12248,24 +12248,22 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12276,7 +12274,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: s_cbranch_execnz .LBB46_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
@@ -12603,24 +12601,22 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12631,7 +12627,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fadd ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -12960,24 +12956,22 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12988,7 +12982,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB48_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 -1024
%result = atomicrmw fadd ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -13308,13 +13302,11 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13655,13 +13647,11 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14028,10 +14018,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v0, v[3:4]
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14307,10 +14295,8 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14635,23 +14621,21 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v4, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_add_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14993,24 +14977,22 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15021,7 +15003,7 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fadd ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -15345,13 +15327,11 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17886,43 +17866,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB68_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
@@ -18217,43 +18189,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB69_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fadd ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18563,43 +18527,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB70_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 -512
%result = atomicrmw fadd ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18881,36 +18837,29 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX7-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v5, v3
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB71_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19201,36 +19150,29 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v5, v3
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB72_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19541,36 +19483,29 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v5, v3
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB73_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19874,43 +19809,35 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB74_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fadd ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -20202,36 +20129,29 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v5, v3
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB75_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20527,43 +20447,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB76_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret <2 x bfloat> %result
@@ -20844,36 +20756,29 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX7-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v5, v3
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB77_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -21168,43 +21073,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB78_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret <2 x bfloat> %result
@@ -21485,36 +21382,29 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX7-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_add_f32_e32 v7, v5, v3
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB79_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index e3fe4b4171517..1230ae8242fa0 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -10015,25 +10015,24 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_max_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10044,7 +10043,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: s_cbranch_execnz .LBB36_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
@@ -10371,25 +10370,24 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_max_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10400,7 +10398,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -10729,25 +10727,24 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_max_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10758,7 +10755,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 -1024
%result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -11065,24 +11062,23 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v4, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_max_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_max_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11412,13 +11408,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11760,13 +11755,12 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12135,9 +12129,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v0, v[3:4]
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12415,9 +12408,8 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12768,25 +12760,24 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_max_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12797,7 +12788,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: s_cbranch_execnz .LBB44_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fmax ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -13121,13 +13112,12 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15437,43 +15427,39 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_max_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
@@ -15891,43 +15877,39 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_max_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16362,43 +16344,39 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_max_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 -512
%result = atomicrmw fmax ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16798,36 +16776,33 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX7-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17236,36 +17211,33 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17697,36 +17669,33 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18154,43 +18123,39 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_max_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fmax ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18601,36 +18566,33 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 1cd0fbe3ebd56..6db9cbf27d448 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -10015,25 +10015,24 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_min_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10044,7 +10043,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: s_cbranch_execnz .LBB36_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret bfloat %result
@@ -10371,25 +10370,24 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_min_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10400,7 +10398,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -10729,25 +10727,24 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_min_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10758,7 +10755,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 -1024
%result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -11065,24 +11062,23 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v4, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_min_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_min_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11412,13 +11408,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11760,13 +11755,12 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12135,9 +12129,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v0, v[3:4]
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12415,9 +12408,8 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12768,25 +12760,24 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_min_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12797,7 +12788,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: s_cbranch_execnz .LBB44_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fmin ptr %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -13121,13 +13112,12 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15437,43 +15427,39 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_min_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
@@ -15891,43 +15877,39 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_min_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16362,43 +16344,39 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_min_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 -512
%result = atomicrmw fmin ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16798,36 +16776,33 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX7-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17236,36 +17211,33 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17697,36 +17669,33 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18154,43 +18123,39 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_min_f32_e32 v3, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v5, v8, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v5, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fmin ptr %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18601,36 +18566,33 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v7, v2
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index f8d2bb98155f0..68c199bccbc95 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9690,24 +9690,22 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -9718,7 +9716,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr %ptr, bfloat %val syncscope("agent") seq_cst
ret bfloat %result
@@ -10045,24 +10043,22 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10073,7 +10069,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fsub ptr %gep, bfloat %val syncscope("agent") seq_cst
@@ -10402,24 +10398,22 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -10430,7 +10424,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: s_cbranch_execnz .LBB34_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 -1024
%result = atomicrmw fsub ptr %gep, bfloat %val syncscope("agent") seq_cst
@@ -10737,23 +10731,21 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v4, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11083,13 +11075,11 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11430,13 +11420,11 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11803,10 +11791,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0x7fe, v0
; GFX7-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v0, v[3:4]
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12082,10 +12068,8 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fe, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12435,24 +12419,22 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
; GFX7-NEXT: flat_load_dword v5, v[0:1]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_not_b32_e32 v3, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v3
+; GFX7-NEXT: v_not_b32_e32 v4, v4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v5, v5, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v7, v6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX7-NEXT: v_and_b32_e32 v7, v6, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX7-NEXT: v_or_b32_e32 v5, v7, v5
; GFX7-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12463,7 +12445,7 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
%result = atomicrmw fsub ptr %gep, bfloat %val seq_cst
@@ -12787,13 +12769,11 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14978,43 +14958,35 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret <2 x bfloat> %result
@@ -15432,43 +15404,35 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fsub ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -15903,43 +15867,35 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB52_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 -512
%result = atomicrmw fsub ptr %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -16339,36 +16295,29 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX7-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v5, v3
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB53_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16777,36 +16726,29 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v5, v3
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17238,36 +17180,29 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v5, v3
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17695,43 +17630,35 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v5, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v6, v2
+; GFX7-NEXT: v_or_b32_e32 v6, v3, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v5, v3, v7, 16
+; GFX7-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds <2 x bfloat>, ptr %ptr, i64 511
%result = atomicrmw fsub ptr %gep, <2 x bfloat> %val seq_cst
@@ -18142,36 +18069,29 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x7fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v3, v[0:1]
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-NEXT: v_sub_f32_e32 v7, v5, v3
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
+; GFX7-NEXT: v_or_b32_e32 v5, v4, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: flat_atomic_cmpswap v6, v[0:1], v[4:5] glc
+; GFX7-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 6004c9c4a41ad..7764a05080d43 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -14030,13 +14030,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -14073,13 +14071,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -14479,13 +14475,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -14523,13 +14517,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -14933,13 +14925,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -14977,13 +14967,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -15361,23 +15349,21 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_add_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: v_mov_b32_e32 v8, v4
; GFX7-NEXT: v_mov_b32_e32 v7, v3
@@ -15403,24 +15389,22 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX6-NEXT: v_not_b32_e32 v5, v3
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX6-NEXT: v_add_f32_e32 v3, v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
; GFX6-NEXT: v_mov_b32_e32 v8, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v3
@@ -15842,13 +15826,11 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -15885,13 +15867,11 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX6-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -16328,13 +16308,11 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -16371,13 +16349,11 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX6-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -16812,10 +16788,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -16847,10 +16821,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -17193,10 +17165,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -17227,10 +17197,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -17634,13 +17602,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX7-NEXT: .LBB62_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -17678,13 +17644,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX6-NEXT: .LBB62_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -18123,13 +18087,11 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB63_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -18166,13 +18128,11 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX6-NEXT: .LBB63_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -21726,45 +21686,37 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB78_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -21775,47 +21727,38 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB78_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
@@ -22124,45 +22067,37 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB79_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -22173,47 +22108,38 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB79_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fadd ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -22522,50 +22448,42 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB80_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -22575,52 +22493,43 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB80_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%result = atomicrmw fadd ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -22920,37 +22829,30 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB81_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -22965,43 +22867,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB81_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
@@ -23302,37 +23196,30 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB82_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23347,43 +23234,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB82_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fadd ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -23684,42 +23563,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB83_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23733,48 +23605,40 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB83_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%unused = atomicrmw fadd ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -24088,45 +23952,37 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB84_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -24137,47 +23993,38 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB84_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fadd ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -24483,37 +24330,30 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB85_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -24528,43 +24368,35 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB85_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fadd ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -24873,45 +24705,37 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB86_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory:
@@ -24922,47 +24746,38 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB86_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret <2 x bfloat> %result
@@ -25261,37 +25076,30 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB87_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -25306,43 +25114,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB87_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
ret void
@@ -25650,45 +25450,37 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB88_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -25699,47 +25491,38 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB88_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret <2 x bfloat> %result
@@ -26038,37 +25821,30 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB89_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26083,43 +25859,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB89_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
ret void
@@ -26427,45 +26195,37 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB90_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__maybe_remote:
@@ -26476,47 +26236,38 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB90_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret <2 x bfloat> %result
@@ -26815,37 +26566,30 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v8, v4, v5
; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB91_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26860,43 +26604,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v8, v4, v5
; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB91_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index dcd0e3aa3f4b0..9623d8178bef5 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8927,13 +8927,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v2
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -8971,13 +8970,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v2
; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -9334,13 +9332,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -9379,13 +9376,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -9745,13 +9741,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -9790,13 +9785,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10133,24 +10127,23 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_max_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_max_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: v_mov_b32_e32 v8, v4
; GFX7-NEXT: v_mov_b32_e32 v7, v3
@@ -10176,25 +10169,24 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX6-NEXT: v_not_b32_e32 v5, v3
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_max_f32_e32 v3, v3, v6
+; GFX6-NEXT: v_max_f32_e32 v3, v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
; GFX6-NEXT: v_mov_b32_e32 v8, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v3
@@ -10531,13 +10523,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -10575,13 +10566,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10931,13 +10921,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -10975,13 +10964,12 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -11355,9 +11343,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11391,9 +11378,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -11676,9 +11662,8 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11711,9 +11696,8 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -12073,13 +12057,12 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -12118,13 +12101,12 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -12478,13 +12460,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -12522,13 +12503,12 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -15203,45 +15183,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15252,47 +15228,42 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB54_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
@@ -15711,45 +15682,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15760,47 +15727,42 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB55_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16219,50 +16181,46 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16272,52 +16230,47 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v5
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB56_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16722,37 +16675,34 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16767,43 +16717,39 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB57_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
@@ -17209,37 +17155,34 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17254,43 +17197,39 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB58_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -17696,42 +17635,39 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17745,48 +17681,44 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v5
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB59_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18211,45 +18143,41 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18260,47 +18188,42 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB60_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18712,37 +18635,34 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18757,43 +18677,39 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_max_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB61_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 96d0c1a9e1128..3b7d637df5a36 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8927,13 +8927,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v2
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -8971,13 +8970,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v2
; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -9334,13 +9332,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -9379,13 +9376,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -9745,13 +9741,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -9790,13 +9785,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10133,24 +10127,23 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_min_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_min_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: v_mov_b32_e32 v8, v4
; GFX7-NEXT: v_mov_b32_e32 v7, v3
@@ -10176,25 +10169,24 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX6-NEXT: v_not_b32_e32 v5, v3
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_min_f32_e32 v3, v3, v6
+; GFX6-NEXT: v_min_f32_e32 v3, v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
; GFX6-NEXT: v_mov_b32_e32 v8, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v3
@@ -10531,13 +10523,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -10575,13 +10566,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10931,13 +10921,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -10975,13 +10964,12 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -11355,9 +11343,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11391,9 +11378,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -11676,9 +11662,8 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11711,9 +11696,8 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v2
; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -12073,13 +12057,12 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -12118,13 +12101,12 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v2
; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -12478,13 +12460,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -12522,13 +12503,12 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -15203,45 +15183,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15252,47 +15228,42 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB54_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %result
@@ -15711,45 +15682,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15760,47 +15727,42 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB55_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16219,50 +16181,46 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16272,52 +16230,47 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v5
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB56_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -16722,37 +16675,34 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16767,43 +16717,39 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB57_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fmin ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
@@ -17209,37 +17155,34 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17254,43 +17197,39 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB58_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -17696,42 +17635,39 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17745,48 +17681,44 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v5
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB59_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18211,45 +18143,41 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB60_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18260,47 +18188,42 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB60_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -18712,37 +18635,34 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX7-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18757,43 +18677,39 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v3
-; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v8, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v8, v2
+; GFX6-NEXT: v_min_f32_e32 v4, v6, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v4, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB61_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fmin ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 9eff7ee37dc54..b72f16a701f2c 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9536,13 +9536,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -9579,13 +9577,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -9941,13 +9937,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -9985,13 +9979,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10350,13 +10342,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -10394,13 +10384,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10736,23 +10724,21 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX7-NEXT: v_not_b32_e32 v5, v3
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX7-NEXT: v_not_b32_e32 v6, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX7-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
; GFX7-NEXT: v_mov_b32_e32 v8, v4
; GFX7-NEXT: v_mov_b32_e32 v7, v3
@@ -10778,24 +10764,22 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
-; GFX6-NEXT: v_not_b32_e32 v5, v3
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
+; GFX6-NEXT: v_not_b32_e32 v6, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX6-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, v4, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
+; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
; GFX6-NEXT: v_mov_b32_e32 v8, v4
; GFX6-NEXT: v_mov_b32_e32 v7, v3
@@ -11132,13 +11116,11 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11175,13 +11157,11 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -11530,13 +11510,11 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11573,13 +11551,11 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -11951,10 +11927,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -11986,10 +11960,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -12270,10 +12242,8 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -12304,10 +12274,8 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -12666,13 +12634,11 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v7, v4
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -12710,13 +12676,11 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v7, v4
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -13069,13 +13033,11 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_not_b32_e32 v5, v5
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -13112,13 +13074,11 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_not_b32_e32 v5, v5
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -15668,45 +15628,37 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16:
@@ -15717,47 +15669,38 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB50_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret <2 x bfloat> %result
@@ -16176,45 +16119,37 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -16225,47 +16160,38 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB51_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -16684,50 +16610,42 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB52_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
@@ -16737,52 +16655,43 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB52_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -17187,37 +17096,30 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB53_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17232,43 +17134,35 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB53_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%unused = atomicrmw fsub ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret void
@@ -17674,37 +17568,30 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17719,43 +17606,35 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB54_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -18161,42 +18040,35 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18210,48 +18082,40 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB55_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
%unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
@@ -18676,45 +18540,37 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -18725,47 +18581,38 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB56_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
@@ -19177,37 +19024,30 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v3, v6
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
+; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19222,43 +19062,35 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v8, v4, v5
; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: v_mov_b32_e32 v7, v5
-; GFX6-NEXT: v_mov_b32_e32 v6, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
+; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
+; GFX6-NEXT: v_alignbit_b32 v6, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v3, v6
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
+; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB57_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
%unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 99a6a5cd346e2..a6bacef2e29a3 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6565,42 +6565,34 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_add_f32_e32 v6, v4, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fadd_ret_v2bf16:
@@ -6608,42 +6600,34 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_add_f32_e32 v6, v4, v3
; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst
ret <2 x bfloat> %result
@@ -6926,42 +6910,34 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_add_f32_e32 v6, v4, v3
; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB25_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fadd_ret_v2bf16__offset:
@@ -6969,44 +6945,36 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v7, v4, v2
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB25_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 16383
%result = atomicrmw fadd ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst
@@ -7280,35 +7248,28 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: ds_read_b32 v3, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v4, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7319,35 +7280,28 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v4, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7624,35 +7578,28 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v4, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7664,36 +7611,29 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_add_f32_e32 v7, v4, v2
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 3de6a80ef14b3..5db923c34990f 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -6458,42 +6458,38 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_max_f32_e32 v2, v6, v3
; GFX7-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fmax_ret_v2bf16:
@@ -6501,42 +6497,38 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_max_f32_e32 v2, v6, v3
; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst
ret <2 x bfloat> %result
@@ -6944,42 +6936,38 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_max_f32_e32 v2, v6, v3
; GFX7-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB25_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fmax_ret_v2bf16__offset:
@@ -6987,44 +6975,40 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v7, v2
+; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB25_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 16383
%result = atomicrmw fmax ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst
@@ -7416,35 +7400,32 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: ds_read_b32 v3, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v6, v2
; GFX7-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7455,35 +7436,32 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v6, v2
; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7878,35 +7856,32 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_max_f32_e32 v3, v6, v2
; GFX7-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7918,36 +7893,33 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_max_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_max_f32_e32 v3, v7, v2
+; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 606a390989cc3..2e7d47bae03d9 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -6458,42 +6458,38 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_min_f32_e32 v2, v6, v3
; GFX7-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fmin_ret_v2bf16:
@@ -6501,42 +6497,38 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_min_f32_e32 v2, v6, v3
; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst
ret <2 x bfloat> %result
@@ -6944,42 +6936,38 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_min_f32_e32 v2, v6, v3
; GFX7-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB25_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fmin_ret_v2bf16__offset:
@@ -6987,44 +6975,40 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v7, v2
+; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB25_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 16383
%result = atomicrmw fmin ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst
@@ -7416,35 +7400,32 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: ds_read_b32 v3, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v6, v2
; GFX7-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7455,35 +7436,32 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v6, v2
; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7878,35 +7856,32 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_min_f32_e32 v3, v6, v2
; GFX7-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v2
-; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7918,36 +7893,33 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v6
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_min_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v4
+; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_min_f32_e32 v3, v7, v2
+; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index a11b5c81f4e83..a371820dfe7d8 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -7278,42 +7278,34 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_sub_f32_e32 v6, v4, v3
; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fsub_ret_v2bf16:
@@ -7321,42 +7313,34 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_sub_f32_e32 v6, v4, v3
; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst
ret <2 x bfloat> %result
@@ -7764,42 +7748,34 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_sub_f32_e32 v6, v4, v3
; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB25_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: local_atomic_fsub_ret_v2bf16__offset:
@@ -7807,44 +7783,36 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_sub_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v7, v4, v2
+; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB25_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 16383
%result = atomicrmw fsub ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst
@@ -8236,35 +8204,28 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: ds_read_b32 v3, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v6, v4, v2
; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8275,35 +8236,28 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v6, v4, v2
; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8698,35 +8652,28 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v0 offset:65532
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_sub_f32_e32 v6, v4, v2
; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
+; GFX7-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3 offset:65532
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8738,36 +8685,29 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX6-NEXT: ds_read_b32 v3, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v5
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v7, v7, v2
-; GFX6-NEXT: v_sub_f32_e32 v6, v6, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_sub_f32_e32 v7, v4, v2
+; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v6, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll b/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll
index d1cbc8e9689ae..bf474695cce8e 100644
--- a/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-phi-s16-fp.ll
@@ -78,19 +78,12 @@ define void @phi_vec1bf16_to_f32(ptr addrspace(1) %src, ptr addrspace(1) %dst) #
; CHECK-NEXT: s_mov_b32 s4, s6
; CHECK-NEXT: s_mov_b32 s5, s6
; CHECK-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
-; CHECK-NEXT: s_mov_b32 s4, 0xffff
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_and_b32_e64 v0, v0, s4
-; CHECK-NEXT: s_mov_b32 s4, 16
-; CHECK-NEXT: v_lshlrev_b32_e64 v0, s4, v0
; CHECK-NEXT: ; %bb.1: ; %bb
-; CHECK-NEXT: v_mul_f32_e64 v0, 1.0, v0
-; CHECK-NEXT: s_mov_b32 s4, 16
-; CHECK-NEXT: v_lshrrev_b32_e64 v0, s4, v0
; CHECK-NEXT: s_mov_b32 s7, 0xf000
; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: s_mov_b32 s4, s6
; CHECK-NEXT: s_mov_b32 s5, s6
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64 offset:2
; CHECK-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -113,15 +106,13 @@ define void @phi_vec1bf16_to_f32_with_const_folding(ptr addrspace(1) %dst) #0 {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: ; %bb.1: ; %bb
-; CHECK-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; CHECK-NEXT: s_mov_b32 s4, 16
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, s4, v2
-; CHECK-NEXT: s_mov_b32 s7, 0xf000
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: s_mov_b32 s4, s6
-; CHECK-NEXT: s_mov_b32 s5, s6
-; CHECK-NEXT: buffer_store_short v2, v[0:1], s[4:7], 0 addr64 offset:2
-; CHECK-NEXT: buffer_store_short v0, v[0:1], s[4:7], 0 addr64
+; CHECK-NEXT: s_mov_b32 s11, 0xf000
+; CHECK-NEXT: s_mov_b32 s10, 0
+; CHECK-NEXT: s_mov_b32 s8, s10
+; CHECK-NEXT: s_mov_b32 s9, s10
+; CHECK-NEXT: v_mov_b32_e32 v2, s4
+; CHECK-NEXT: buffer_store_short v2, v[0:1], s[8:11], 0 addr64 offset:2
+; CHECK-NEXT: buffer_store_short v0, v[0:1], s[8:11], 0 addr64
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
entry:
>From e6df7bc2e539d7d286984b651bb930cb786bf13f Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 20 Sep 2026 13:43:15 +0200
Subject: [PATCH 3/5] update other tests
---
llvm/test/CodeGen/Generic/bfloat.ll | 3 +-
.../CodeGen/Hexagon/autohvx/bf16insert.ll | 91 +-
llvm/test/CodeGen/Hexagon/bfloat.ll | 130 +-
.../fixed-vector-convert-from-arbitrary-fp.ll | 17 +-
.../RISCV/rvv/fixed-vectors-scalarized.ll | 230 +-
llvm/test/CodeGen/X86/atomic-load-store.ll | 1886 +++++------------
llvm/test/CodeGen/X86/atomic-non-integer.ll | 48 +-
llvm/test/CodeGen/X86/ldexp.ll | 8 +-
llvm/test/CodeGen/X86/llvm.frexp.ll | 19 +-
llvm/test/CodeGen/X86/select-phi-s16-fp.ll | 24 +-
10 files changed, 707 insertions(+), 1749 deletions(-)
diff --git a/llvm/test/CodeGen/Generic/bfloat.ll b/llvm/test/CodeGen/Generic/bfloat.ll
index 5f67dcca669fd..5c41e2fc57178 100644
--- a/llvm/test/CodeGen/Generic/bfloat.ll
+++ b/llvm/test/CodeGen/Generic/bfloat.ll
@@ -34,7 +34,7 @@
; FIXME: s390x crashes
; FIXME: ve crashes
; FIXME: wasm crashes
-; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,BAD %}
+; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK %}
; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK %}
; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK %}
; RUN: %if xcore-registered-target %{ llc %s -o - -mtriple=xcore-unknown-unknown | FileCheck %s --check-prefixes=ALL,CHECK %}
@@ -63,7 +63,6 @@ define i16 @to_bits(bfloat %f) nounwind {
; CHECK-NOT: __extend
; CHECK-NOT: __trunc
; CHECK-NOT: __gnu
-; BAD: __truncsfbf2
%bits = bitcast bfloat %f to i16
ret i16 %bits
}
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/bf16insert.ll b/llvm/test/CodeGen/Hexagon/autohvx/bf16insert.ll
index 23234e15e4e61..5e0e0afd8eb83 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/bf16insert.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/bf16insert.ll
@@ -10,42 +10,23 @@ define <64 x bfloat> @insert_bf16_var(<64 x bfloat> %v, bfloat %a, i32 %i) #0 {
; CHECK-LABEL: insert_bf16_var:
; CHECK: // %bb.0:
; CHECK-NEXT: {
-; CHECK-NEXT: r3 = extractu(r0,#17,#0)
-; CHECK-NEXT: r2 = ##32768
-; CHECK-NEXT: r4 = and(r1,#1)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
; CHECK-NEXT: r1 = and(##-4,asl(r1,#1))
-; CHECK-NEXT: p1 = cmp.eq(r3,r2)
-; CHECK-NEXT: r2 = and(r0,r2)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: p0 = sfclass(r0,#16)
-; CHECK-NEXT: r3 = asrh(r0)
-; CHECK-NEXT: r6 = add(r0,r2)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r4 = asl(r4,#4)
-; CHECK-NEXT: if (p1) r2 = zxth(r3)
-; CHECK-NEXT: r5 = #16
-; CHECK-NEXT: r0 = asrh(r6)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r3 = vextract(v0,r1)
+; CHECK-NEXT: r3 = #16
+; CHECK-NEXT: r2 = and(r1,#1)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: if (!p1) r2 = zxth(r0)
+; CHECK-NEXT: r2 = asl(r2,#4)
; CHECK-NEXT: v31 = vror(v0,r1)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: if (p0) r2 = ##32767
+; CHECK-NEXT: r4 = vextract(v0,r1)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: r3 = insert(r2,r5:4)
+; CHECK-NEXT: r4 = insert(r0,r3:2)
; CHECK-NEXT: r2 = sub(#128,r1)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: v31.w = vinsert(r3)
+; CHECK-NEXT: v31.w = vinsert(r4)
; CHECK-NEXT: }
; CHECK-NEXT: {
; CHECK-NEXT: v0 = vror(v31,r2)
@@ -59,36 +40,15 @@ define <64 x bfloat> @insert_bf16_const(<64 x bfloat> %v, bfloat %a) #0 {
; CHECK-LABEL: insert_bf16_const:
; CHECK: // %bb.0:
; CHECK-NEXT: {
-; CHECK-NEXT: r2 = extractu(r0,#17,#0)
-; CHECK-NEXT: r1 = ##32768
-; CHECK-NEXT: r5 = asrh(r0)
+; CHECK-NEXT: r1 = #32
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: p0 = sfclass(r0,#16)
-; CHECK-NEXT: r4 = and(r0,r1)
-; CHECK-NEXT: p1 = cmp.eq(r2,r1)
-; CHECK-NEXT: r3 = #32
+; CHECK-NEXT: r2 = vextract(v0,r1)
; CHECK-NEXT: }
; CHECK-NEXT: {
+; CHECK-NEXT: r0 = combine(r0.l,r2.l)
; CHECK-NEXT: r2 = #96
-; CHECK-NEXT: r6 = add(r0,r4)
-; CHECK-NEXT: if (p1) r1 = zxth(r5)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r4 = vextract(v0,r3)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = asrh(r6)
-; CHECK-NEXT: v31 = vror(v0,r3)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: if (!p1) r1 = zxth(r0)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: if (p0) r1 = ##32767
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = combine(r1.l,r4.l)
+; CHECK-NEXT: v31 = vror(v0,r1)
; CHECK-NEXT: }
; CHECK-NEXT: {
; CHECK-NEXT: v31.w = vinsert(r0)
@@ -105,42 +65,21 @@ define void @insert_bf16_distinct_dest(ptr %p, ptr %q, bfloat %a, i32 %i) #0 {
; CHECK-LABEL: insert_bf16_distinct_dest:
; CHECK: // %bb.0:
; CHECK-NEXT: {
-; CHECK-NEXT: r5 = extractu(r2,#17,#0)
-; CHECK-NEXT: r4 = ##32768
-; CHECK-NEXT: r6 = and(r3,#1)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
; CHECK-NEXT: r3 = and(##-4,asl(r3,#1))
-; CHECK-NEXT: p0 = cmp.eq(r5,r4)
-; CHECK-NEXT: r4 = and(r2,r4)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r5 = asrh(r2)
-; CHECK-NEXT: r4 = add(r2,r4)
+; CHECK-NEXT: r4 = and(r3,#1)
; CHECK-NEXT: v0 = vmem(r0+#0)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: if (p0) r5 = zxth(r5)
+; CHECK-NEXT: r4 = asl(r4,#4)
+; CHECK-NEXT: r5 = #16
; CHECK-NEXT: }
; CHECK-NEXT: {
; CHECK-NEXT: r0 = vextract(v0,r3)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: r6 = asl(r6,#4)
-; CHECK-NEXT: r4 = asrh(r4)
-; CHECK-NEXT: r7 = #16
-; CHECK-NEXT: v0 = vror(v0,r3)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: p0 = sfclass(r2,#16)
-; CHECK-NEXT: if (!p0) r5 = zxth(r4)
+; CHECK-NEXT: r0 = insert(r2,r5:4)
; CHECK-NEXT: r2 = sub(#128,r3)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: if (p0) r5 = ##32767
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = insert(r5,r7:6)
+; CHECK-NEXT: v0 = vror(v0,r3)
; CHECK-NEXT: }
; CHECK-NEXT: {
; CHECK-NEXT: v0.w = vinsert(r0)
diff --git a/llvm/test/CodeGen/Hexagon/bfloat.ll b/llvm/test/CodeGen/Hexagon/bfloat.ll
index 550a2b54b2dea..596c0f0dc0db4 100644
--- a/llvm/test/CodeGen/Hexagon/bfloat.ll
+++ b/llvm/test/CodeGen/Hexagon/bfloat.ll
@@ -6,10 +6,7 @@ define bfloat @load_scalar_bf(ptr %addr) {
; CHECK: .cfi_startproc
; CHECK-NEXT: // %bb.0: // %entry
; CHECK-NEXT: {
-; CHECK-NEXT: [[REG:r[0-9]+]] = memuh(r0+#0)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = aslh([[REG]])
+; CHECK-NEXT: r0 = memuh(r0+#0)
; CHECK-NEXT: jumpr r31
; CHECK-NEXT: }
@@ -23,35 +20,8 @@ define void @store_scalar_bf(bfloat %v, ptr %addr) {
; CHECK: .cfi_startproc
; CHECK-NEXT: // %bb.0: // %entry
; CHECK-NEXT: {
-; CHECK-NEXT: [[R_A:r[0-9]+]] = ##131071
-; CHECK-NEXT: [[R_B:r[0-9]+]] = ##32768
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[R_C:r[0-9]+]] = ##65535
-; CHECK-NEXT: [[R_D:r[0-9]+]] = asrh(r0)
-; CHECK-NEXT: [[R_A]] = and(r0,[[R_A]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[P0:p[0-9]+]] = sfclass(r0,#16)
-; CHECK-NEXT: [[R_E:r[0-9]+]] = and(r0,[[R_B]])
-; CHECK-NEXT: [[P1:p[0-9]+]] = cmp.eq([[R_A]],[[R_B]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = add(r0,[[R_E]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = asrh(r0)
-; CHECK-NEXT: if ([[P1]]) [[R_B]] = and([[R_D]],[[R_C]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: if (![[P1]]) [[R_B]] = and(r0,[[R_C]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: if ([[P0]]) [[R_B]] = ##32767
-; CHECK-NEXT: }
-; CHECK-NEXT: {
; CHECK-NEXT: jumpr r31
-; CHECK-NEXT: memh(r1+#0) = [[R_B]]
+; CHECK-NEXT: memh(r1+#0) = r0
; CHECK-NEXT: }
entry:
@@ -64,74 +34,36 @@ define bfloat @sum(bfloat %a, bfloat %b) #0 {
; CHECK: .cfi_startproc
; CHECK-NEXT: // %bb.0: // %entry
; CHECK-NEXT: {
-; CHECK-NEXT: [[R3:r[0-9]+]] = ##32768
-; CHECK-NEXT: [[R4:r[0-9]+]] = ##131071
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[R2:r[0-9]+]] = ##65535
-; CHECK-NEXT: [[R6:r[0-9]+]] = and(r0,[[R3]])
-; CHECK-NEXT: [[R5:r[0-9]+]] = and(r0,[[R4]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[R8:r[0-9]+]] = and(r1,[[R3]])
-; CHECK-NEXT: [[R7:r[0-9]+]] = and(r1,[[R4]])
-; CHECK-NEXT: [[R6]] = add(r0,[[R6]])
-; CHECK-NEXT: [[P0:p[0-9]+]] = cmp.eq([[R5]],[[R3]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[R5]] = asrh(r0)
-; CHECK-NEXT: [[P1:p[0-9]+]] = cmp.eq([[R7]],[[R3]])
-; CHECK-NEXT: [[R7]] = asrh(r1)
-; CHECK-NEXT: [[R8]] = add(r1,[[R8]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[R6]] = asrh([[R6]])
-; CHECK-NEXT: if ([[P0]]) [[R5]] = and([[R5]],[[R2]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[P0]] = sfclass(r0,#16)
-; CHECK-NEXT: if (![[P0]]) [[R5]] = and([[R6]],[[R2]])
-; CHECK-NEXT: [[R6]] = asrh([[R8]])
-; CHECK-NEXT: if ([[P1]]) [[R7]] = and([[R7]],[[R2]])
+; CHECK-NEXT: r0 = aslh(r0)
+; CHECK-NEXT: r1 = aslh(r1)
+; CHECK-NEXT: r2 = ##32768
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[P1]] = sfclass(r1,#16)
-; CHECK-NEXT: [[R0:r[0-9]+]] = #32767
-; CHECK-NEXT: if (![[P1]]) [[R7]] = and([[R6]],[[R2]])
+; CHECK-NEXT: r0 = sfadd(r0,r1)
+; CHECK-NEXT: r6 = ##131071
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: if ([[P0]]) [[R5]] = add([[R0]],#0)
-; CHECK-NEXT: if ([[P1]]) [[R7]] = add([[R0]],#0)
+; CHECK-NEXT: r3 = ##65535
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[R6]] = aslh([[R7]])
-; CHECK-NEXT: [[R5]] = aslh([[R5]])
+; CHECK-NEXT: p0 = sfclass(r0,#16)
+; CHECK-NEXT: r1 = and(r0,r6)
+; CHECK-NEXT: r4 = and(r0,r2)
+; CHECK-NEXT: r5 = asrh(r0)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[R1:r[0-9]+]] = sfadd([[R5]],[[R6]])
+; CHECK-NEXT: r0 = add(r0,r4)
+; CHECK-NEXT: p1 = cmp.eq(r1,r2)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[R5]] = and([[R1]],[[R3]])
-; CHECK-NEXT: [[R4]] = and([[R1]],[[R4]])
-; CHECK-NEXT: [[R6]] = asrh([[R1]])
+; CHECK-NEXT: r1 = asrh(r0)
+; CHECK-NEXT: if (p1) r0 = and(r5,r3)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[P0]] = cmp.eq([[R4]],[[R3]])
-; CHECK-NEXT: [[R5]] = add([[R1]],[[R5]])
+; CHECK-NEXT: if (!p1) r0 = and(r1,r3)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[R3]] = asrh([[R5]])
-; CHECK-NEXT: if ([[P0]]) [[R4]] = and([[R6]],[[R2]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: [[P0]] = sfclass([[R1]],#16)
-; CHECK-NEXT: if (![[P0]]) [[R4]] = and([[R3]],[[R2]])
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: if ([[P0]]) [[R4]] = add([[R0]],#0)
-; CHECK-NEXT: }
-; CHECK-NEXT: {
-; CHECK-NEXT: r0 = aslh([[R4]])
+; CHECK-NEXT: if (p0) r0 = ##32767
; CHECK-NEXT: jumpr r31
; CHECK-NEXT: }
@@ -146,28 +78,32 @@ define dso_local i32 @double_bf16(bfloat %a, bfloat %b) #0 {
; CHECK: .cfi_startproc
; CHECK-NEXT: // %bb.0: // %entry
; CHECK-NEXT: {
-; CHECK-NEXT: [[PAIR_A:r[0-9]+:[0-9]+]] = convert_sf2df(r0)
-; CHECK-NEXT: [[PAIR_B:r[0-9]+:[0-9]+]] = convert_sf2df(r1)
-; CHECK-NEXT: [[SP:r[0-9]+]] = add([[SP]],#-16)
+; CHECK-NEXT: r0 = aslh(r0)
+; CHECK-NEXT: r1 = aslh(r1)
+; CHECK-NEXT: r29 = add(r29,#-16)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r3:2 = convert_sf2df(r0)
+; CHECK-NEXT: r5:4 = convert_sf2df(r1)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[RESULT:r[0-9]+:[0-9]+]] = dfadd([[PAIR_A]],[[PAIR_B]])
+; CHECK-NEXT: r3:2 = dfadd(r3:2,r5:4)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: memd([[SP]]+#8) = [[RESULT]]
+; CHECK-NEXT: memd(r29+#8) = r3:2
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[TMP:r[0-9]+]] = convert_df2sf([[RESULT]])
+; CHECK-NEXT: r4 = convert_df2sf(r3:2)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[TMP1:r[0-9]+]] = lsr([[TMP]],#16)
-; CHECK-NEXT: [[R0:r[0-9]+]] = and([[TMP]],##-65536)
-; CHECK-NEXT: memh([[SP]]+#6) = [[TMP1]].new
+; CHECK-NEXT: r4 = insert(r1,#16,#0)
+; CHECK-NEXT: r2 = lsr(r4,#16)
+; CHECK-NEXT: r29 = add(r29,#16)
+; CHECK-NEXT: memh(r29+#6) = r2.new
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: [[R0]] = convert_sf2w([[R0]]):chop
+; CHECK-NEXT: r0 = convert_sf2w(r4):chop
; CHECK-NEXT: jumpr r31
-; CHECK-NEXT: [[SP]] = add([[SP]],#16)
; CHECK-NEXT: }
entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vector-convert-from-arbitrary-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vector-convert-from-arbitrary-fp.ll
index 1e39b4d846831..263418a47dea3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vector-convert-from-arbitrary-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vector-convert-from-arbitrary-fp.ll
@@ -116,20 +116,15 @@ define <2 x bfloat> @v2fp8e5m2_to_v2bf16(<2 x i8> %x) nounwind {
; CHECK-NEXT: flw fa0, 12(sp)
; CHECK-NEXT: zext.h s0, a0
; CHECK-NEXT: call __truncsfbf2
-; CHECK-NEXT: lui a0, 1048560
-; CHECK-NEXT: fmv.x.w a1, fa0
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: or a1, s0, a1
+; CHECK-NEXT: fmv.x.w a0, fa0
+; CHECK-NEXT: slli a0, a0, 16
+; CHECK-NEXT: or a0, s0, a0
; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT: vmv.s.x v8, a1
+; CHECK-NEXT: vmv.s.x v8, a0
; CHECK-NEXT: vsetivli zero, 1, e16, mf4, ta, ma
-; CHECK-NEXT: vmv.x.s a1, v8
+; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: vslidedown.vi v8, v8, 1
-; CHECK-NEXT: vmv.x.s a2, v8
-; CHECK-NEXT: or a1, a1, a0
-; CHECK-NEXT: or a0, a2, a0
-; CHECK-NEXT: fmv.w.x fa0, a1
-; CHECK-NEXT: fmv.w.x fa1, a0
+; CHECK-NEXT: vmv.x.s a1, v8
; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
; CHECK-NEXT: addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll
index 92ec298fbd668..6525456695f72 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-scalarized.ll
@@ -1,88 +1,166 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,RV64
define <8 x float> @fpext_v8bf16(<8 x bfloat> %x) {
-; CHECK-LABEL: fpext_v8bf16:
-; CHECK: # %bb.0:
-; CHECK-NEXT: fmv.x.w a0, fa7
-; CHECK-NEXT: fmv.x.w a1, fa6
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT: vmv.s.x v9, a0
-; CHECK-NEXT: fmv.x.w a0, fa5
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v11, a1
-; CHECK-NEXT: fmv.x.w a1, fa4
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vmv.s.x v12, a0
-; CHECK-NEXT: fmv.x.w a0, fa3
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v10, a1
-; CHECK-NEXT: fmv.x.w a1, fa2
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vmv.s.x v13, a0
-; CHECK-NEXT: fmv.x.w a0, fa1
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v14, a1
-; CHECK-NEXT: fmv.x.w a1, fa0
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vmv.s.x v15, a0
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v8, a1
-; CHECK-NEXT: vslideup.vi v14, v13, 1
-; CHECK-NEXT: vslideup.vi v8, v15, 1
-; CHECK-NEXT: vslideup.vi v11, v9, 1
-; CHECK-NEXT: vslideup.vi v10, v12, 1
-; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT: vslideup.vi v8, v14, 2
-; CHECK-NEXT: vslideup.vi v10, v11, 2
-; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-NEXT: vslideup.vi v8, v10, 4
-; CHECK-NEXT: ret
+; RV32-LABEL: fpext_v8bf16:
+; RV32: # %bb.0:
+; RV32-NEXT: lhu a1, 28(a0)
+; RV32-NEXT: lhu a2, 24(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT: vmv.s.x v9, a1
+; RV32-NEXT: lhu a1, 20(a0)
+; RV32-NEXT: slli a2, a2, 16
+; RV32-NEXT: vmv.s.x v11, a2
+; RV32-NEXT: lhu a2, 16(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vmv.s.x v12, a1
+; RV32-NEXT: lhu a1, 12(a0)
+; RV32-NEXT: slli a2, a2, 16
+; RV32-NEXT: vmv.s.x v10, a2
+; RV32-NEXT: lhu a2, 8(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vmv.s.x v13, a1
+; RV32-NEXT: lhu a1, 4(a0)
+; RV32-NEXT: slli a2, a2, 16
+; RV32-NEXT: vmv.s.x v14, a2
+; RV32-NEXT: lhu a0, 0(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vmv.s.x v15, a1
+; RV32-NEXT: slli a0, a0, 16
+; RV32-NEXT: vmv.s.x v8, a0
+; RV32-NEXT: vslideup.vi v14, v13, 1
+; RV32-NEXT: vslideup.vi v8, v15, 1
+; RV32-NEXT: vslideup.vi v11, v9, 1
+; RV32-NEXT: vslideup.vi v10, v12, 1
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vslideup.vi v8, v14, 2
+; RV32-NEXT: vslideup.vi v10, v11, 2
+; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT: vslideup.vi v8, v10, 4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: fpext_v8bf16:
+; RV64: # %bb.0:
+; RV64-NEXT: lhu a1, 56(a0)
+; RV64-NEXT: lhu a2, 48(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT: vmv.s.x v9, a1
+; RV64-NEXT: lhu a1, 40(a0)
+; RV64-NEXT: slli a2, a2, 16
+; RV64-NEXT: vmv.s.x v11, a2
+; RV64-NEXT: lhu a2, 32(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vmv.s.x v12, a1
+; RV64-NEXT: lhu a1, 24(a0)
+; RV64-NEXT: slli a2, a2, 16
+; RV64-NEXT: vmv.s.x v10, a2
+; RV64-NEXT: lhu a2, 16(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vmv.s.x v13, a1
+; RV64-NEXT: lhu a1, 8(a0)
+; RV64-NEXT: slli a2, a2, 16
+; RV64-NEXT: vmv.s.x v14, a2
+; RV64-NEXT: lhu a0, 0(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vmv.s.x v15, a1
+; RV64-NEXT: slli a0, a0, 16
+; RV64-NEXT: vmv.s.x v8, a0
+; RV64-NEXT: vslideup.vi v14, v13, 1
+; RV64-NEXT: vslideup.vi v8, v15, 1
+; RV64-NEXT: vslideup.vi v11, v9, 1
+; RV64-NEXT: vslideup.vi v10, v12, 1
+; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT: vslideup.vi v8, v14, 2
+; RV64-NEXT: vslideup.vi v10, v11, 2
+; RV64-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT: vslideup.vi v8, v10, 4
+; RV64-NEXT: ret
%y = fpext <8 x bfloat> %x to <8 x float>
ret <8 x float> %y
}
define <8 x float> @fpext_v8f16(<8 x bfloat> %x) {
-; CHECK-LABEL: fpext_v8f16:
-; CHECK: # %bb.0:
-; CHECK-NEXT: fmv.x.w a0, fa7
-; CHECK-NEXT: fmv.x.w a1, fa6
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT: vmv.s.x v9, a0
-; CHECK-NEXT: fmv.x.w a0, fa5
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v11, a1
-; CHECK-NEXT: fmv.x.w a1, fa4
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vmv.s.x v12, a0
-; CHECK-NEXT: fmv.x.w a0, fa3
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v10, a1
-; CHECK-NEXT: fmv.x.w a1, fa2
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vmv.s.x v13, a0
-; CHECK-NEXT: fmv.x.w a0, fa1
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v14, a1
-; CHECK-NEXT: fmv.x.w a1, fa0
-; CHECK-NEXT: slli a0, a0, 16
-; CHECK-NEXT: vmv.s.x v15, a0
-; CHECK-NEXT: slli a1, a1, 16
-; CHECK-NEXT: vmv.s.x v8, a1
-; CHECK-NEXT: vslideup.vi v14, v13, 1
-; CHECK-NEXT: vslideup.vi v8, v15, 1
-; CHECK-NEXT: vslideup.vi v11, v9, 1
-; CHECK-NEXT: vslideup.vi v10, v12, 1
-; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT: vslideup.vi v8, v14, 2
-; CHECK-NEXT: vslideup.vi v10, v11, 2
-; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-NEXT: vslideup.vi v8, v10, 4
-; CHECK-NEXT: ret
+; RV32-LABEL: fpext_v8f16:
+; RV32: # %bb.0:
+; RV32-NEXT: lhu a1, 28(a0)
+; RV32-NEXT: lhu a2, 24(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT: vmv.s.x v9, a1
+; RV32-NEXT: lhu a1, 20(a0)
+; RV32-NEXT: slli a2, a2, 16
+; RV32-NEXT: vmv.s.x v11, a2
+; RV32-NEXT: lhu a2, 16(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vmv.s.x v12, a1
+; RV32-NEXT: lhu a1, 12(a0)
+; RV32-NEXT: slli a2, a2, 16
+; RV32-NEXT: vmv.s.x v10, a2
+; RV32-NEXT: lhu a2, 8(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vmv.s.x v13, a1
+; RV32-NEXT: lhu a1, 4(a0)
+; RV32-NEXT: slli a2, a2, 16
+; RV32-NEXT: vmv.s.x v14, a2
+; RV32-NEXT: lhu a0, 0(a0)
+; RV32-NEXT: slli a1, a1, 16
+; RV32-NEXT: vmv.s.x v15, a1
+; RV32-NEXT: slli a0, a0, 16
+; RV32-NEXT: vmv.s.x v8, a0
+; RV32-NEXT: vslideup.vi v14, v13, 1
+; RV32-NEXT: vslideup.vi v8, v15, 1
+; RV32-NEXT: vslideup.vi v11, v9, 1
+; RV32-NEXT: vslideup.vi v10, v12, 1
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vslideup.vi v8, v14, 2
+; RV32-NEXT: vslideup.vi v10, v11, 2
+; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT: vslideup.vi v8, v10, 4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: fpext_v8f16:
+; RV64: # %bb.0:
+; RV64-NEXT: lhu a1, 56(a0)
+; RV64-NEXT: lhu a2, 48(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT: vmv.s.x v9, a1
+; RV64-NEXT: lhu a1, 40(a0)
+; RV64-NEXT: slli a2, a2, 16
+; RV64-NEXT: vmv.s.x v11, a2
+; RV64-NEXT: lhu a2, 32(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vmv.s.x v12, a1
+; RV64-NEXT: lhu a1, 24(a0)
+; RV64-NEXT: slli a2, a2, 16
+; RV64-NEXT: vmv.s.x v10, a2
+; RV64-NEXT: lhu a2, 16(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vmv.s.x v13, a1
+; RV64-NEXT: lhu a1, 8(a0)
+; RV64-NEXT: slli a2, a2, 16
+; RV64-NEXT: vmv.s.x v14, a2
+; RV64-NEXT: lhu a0, 0(a0)
+; RV64-NEXT: slli a1, a1, 16
+; RV64-NEXT: vmv.s.x v15, a1
+; RV64-NEXT: slli a0, a0, 16
+; RV64-NEXT: vmv.s.x v8, a0
+; RV64-NEXT: vslideup.vi v14, v13, 1
+; RV64-NEXT: vslideup.vi v8, v15, 1
+; RV64-NEXT: vslideup.vi v11, v9, 1
+; RV64-NEXT: vslideup.vi v10, v12, 1
+; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT: vslideup.vi v8, v14, 2
+; RV64-NEXT: vslideup.vi v10, v11, 2
+; RV64-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT: vslideup.vi v8, v10, 4
+; RV64-NEXT: ret
%y = fpext <8 x bfloat> %x to <8 x float>
ret <8 x float> %y
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 41131b28d9b23..c959a7b2d40ea 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -237,28 +237,16 @@ define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: pushq %rax
-; CHECK-SSE-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
-; CHECK-SSE-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT: movw %ax, (%rdi)
-; CHECK-SSE-O0-NEXT: popq %rax
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: pushq %rax
-; CHECK-AVX-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT: movw %ax, (%rdi)
-; CHECK-AVX-O0-NEXT: popq %rax
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x bfloat> %v, ptr %x release, align 2
ret void
@@ -873,97 +861,29 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $24, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-SSE2-O0-NEXT: movl %eax, (%rdi)
-; CHECK-SSE2-O0-NEXT: addq $24, %rsp
-; CHECK-SSE2-O0-NEXT: retq
-;
-; CHECK-SSE4-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $24, %rsp
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-SSE4-O0-NEXT: movl %eax, (%rdi)
-; CHECK-SSE4-O0-NEXT: addq $24, %rsp
-; CHECK-SSE4-O0-NEXT: retq
-;
-; CHECK-AVX2-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $24, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-AVX2-O0-NEXT: movl %eax, (%rdi)
-; CHECK-AVX2-O0-NEXT: addq $24, %rsp
-; CHECK-AVX2-O0-NEXT: retq
+; CHECK-SSE-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-SSE-O0: # %bb.0:
+; CHECK-SSE-O0-NEXT: pextrw $1, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movd %xmm0, %ecx
+; CHECK-SSE-O0-NEXT: # kill: def $cx killed $cx killed $ecx
+; CHECK-SSE-O0-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax
+; CHECK-SSE-O0-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O0-NEXT: retq
;
-; CHECK-AVX512-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $24, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-AVX512-O0-NEXT: movl %eax, (%rdi)
-; CHECK-AVX512-O0-NEXT: addq $24, %rsp
-; CHECK-AVX512-O0-NEXT: retq
+; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-AVX-O0: # %bb.0:
+; CHECK-AVX-O0-NEXT: vpextrw $1, %xmm0, %eax
+; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, %ecx
+; CHECK-AVX-O0-NEXT: # kill: def $cx killed $cx killed $ecx
+; CHECK-AVX-O0-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax
+; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O0-NEXT: retq
store atomic <2 x bfloat> %v, ptr %x release, align 4
ret void
}
@@ -1027,171 +947,41 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $40, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-SSE2-O0-NEXT: movq %rax, (%rdi)
-; CHECK-SSE2-O0-NEXT: addq $40, %rsp
-; CHECK-SSE2-O0-NEXT: retq
-;
-; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $40, %rsp
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm3
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-SSE4-O0-NEXT: movq %rax, (%rdi)
-; CHECK-SSE4-O0-NEXT: addq $40, %rsp
-; CHECK-SSE4-O0-NEXT: retq
-;
-; CHECK-AVX2-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $40, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-AVX2-O0-NEXT: movq %rax, (%rdi)
-; CHECK-AVX2-O0-NEXT: addq $40, %rsp
-; CHECK-AVX2-O0-NEXT: retq
+; CHECK-SSE-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-SSE-O0: # %bb.0:
+; CHECK-SSE-O0-NEXT: pextrw $3, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: pextrw $2, %xmm0, %ecx
+; CHECK-SSE-O0-NEXT: # kill: def $cx killed $cx killed $ecx
+; CHECK-SSE-O0-NEXT: pextrw $1, %xmm0, %edx
+; CHECK-SSE-O0-NEXT: # kill: def $dx killed $dx killed $edx
+; CHECK-SSE-O0-NEXT: movd %xmm0, %esi
+; CHECK-SSE-O0-NEXT: # kill: def $si killed $si killed $esi
+; CHECK-SSE-O0-NEXT: movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax
+; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: retq
;
-; CHECK-AVX512-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $40, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-AVX512-O0-NEXT: movq %rax, (%rdi)
-; CHECK-AVX512-O0-NEXT: addq $40, %rsp
-; CHECK-AVX512-O0-NEXT: retq
+; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-AVX-O0: # %bb.0:
+; CHECK-AVX-O0-NEXT: vpextrw $3, %xmm0, %eax
+; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX-O0-NEXT: vpextrw $2, %xmm0, %ecx
+; CHECK-AVX-O0-NEXT: # kill: def $cx killed $cx killed $ecx
+; CHECK-AVX-O0-NEXT: vpextrw $1, %xmm0, %edx
+; CHECK-AVX-O0-NEXT: # kill: def $dx killed $dx killed $edx
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, %esi
+; CHECK-AVX-O0-NEXT: # kill: def $si killed $si killed $esi
+; CHECK-AVX-O0-NEXT: movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-O0-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-O0-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax
+; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: retq
store atomic <4 x bfloat> %v, ptr %x release, align 8
ret void
}
@@ -2693,593 +2483,235 @@ define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounw
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
-; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $120, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: xorps %xmm13, %xmm13
-; CHECK-SSE2-O0-NEXT: movaps %xmm13, %xmm15
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
-; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm14
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm11
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm11
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm7
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: movl $32, %edi
-; CHECK-SSE2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
-; CHECK-SSE2-O0-NEXT: movl $3, %ecx
-; CHECK-SSE2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE2-O0-NEXT: addq $120, %rsp
-; CHECK-SSE2-O0-NEXT: retq
-;
-; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
-; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $120, %rsp
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm15
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm14
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm12
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm11
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm13
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm7
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: movl $32, %edi
-; CHECK-SSE4-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
-; CHECK-SSE4-O0-NEXT: movl $3, %ecx
-; CHECK-SSE4-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE4-O0-NEXT: addq $120, %rsp
-; CHECK-SSE4-O0-NEXT: retq
+; CHECK-SSE-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
+; CHECK-SSE-O0: # %bb.0:
+; CHECK-SSE-O0-NEXT: pushq %rbp
+; CHECK-SSE-O0-NEXT: pushq %r15
+; CHECK-SSE-O0-NEXT: pushq %r14
+; CHECK-SSE-O0-NEXT: pushq %r13
+; CHECK-SSE-O0-NEXT: pushq %r12
+; CHECK-SSE-O0-NEXT: pushq %rbx
+; CHECK-SSE-O0-NEXT: subq $56, %rsp
+; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
+; CHECK-SSE-O0-NEXT: pextrw $7, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-SSE-O0-NEXT: pextrw $6, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %cx
+; CHECK-SSE-O0-NEXT: pextrw $5, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %dx
+; CHECK-SSE-O0-NEXT: pextrw $4, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %di
+; CHECK-SSE-O0-NEXT: pextrw $3, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r8w
+; CHECK-SSE-O0-NEXT: pextrw $2, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r9w
+; CHECK-SSE-O0-NEXT: pextrw $1, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r10w
+; CHECK-SSE-O0-NEXT: movd %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r11w
+; CHECK-SSE-O0-NEXT: pextrw $7, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %bx
+; CHECK-SSE-O0-NEXT: pextrw $6, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %bp
+; CHECK-SSE-O0-NEXT: pextrw $5, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r14w
+; CHECK-SSE-O0-NEXT: pextrw $4, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r15w
+; CHECK-SSE-O0-NEXT: pextrw $3, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r12w
+; CHECK-SSE-O0-NEXT: pextrw $2, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r13w
+; CHECK-SSE-O0-NEXT: pextrw $1, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-SSE-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-SSE-O0-NEXT: movw %r13w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r12w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r15w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r14w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %bp, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r11w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r10w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r8w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %di, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %cx, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movl $32, %edi
+; CHECK-SSE-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-SSE-O0-NEXT: movl $3, %ecx
+; CHECK-SSE-O0-NEXT: callq __atomic_store at PLT
+; CHECK-SSE-O0-NEXT: addq $56, %rsp
+; CHECK-SSE-O0-NEXT: popq %rbx
+; CHECK-SSE-O0-NEXT: popq %r12
+; CHECK-SSE-O0-NEXT: popq %r13
+; CHECK-SSE-O0-NEXT: popq %r14
+; CHECK-SSE-O0-NEXT: popq %r15
+; CHECK-SSE-O0-NEXT: popq %rbp
+; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $120, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm8
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
+; CHECK-AVX2-O0-NEXT: pushq %rbp
+; CHECK-AVX2-O0-NEXT: pushq %r15
+; CHECK-AVX2-O0-NEXT: pushq %r14
+; CHECK-AVX2-O0-NEXT: pushq %r13
+; CHECK-AVX2-O0-NEXT: pushq %r12
+; CHECK-AVX2-O0-NEXT: pushq %rbx
+; CHECK-AVX2-O0-NEXT: subq $56, %rsp
+; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %cx
+; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %dx
+; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %di
+; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r8w
+; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r9w
+; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r10w
+; CHECK-AVX2-O0-NEXT: vmovd %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r11w
; CHECK-AVX2-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vzeroupper
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %bx
+; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %bp
+; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r14w
+; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r15w
+; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r12w
+; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r13w
+; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovd %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX2-O0-NEXT: movw %r13w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r12w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r15w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r14w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %bp, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r11w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r10w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r8w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %di, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %cx, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $32, %edi
; CHECK-AVX2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
+; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX2-O0-NEXT: addq $120, %rsp
+; CHECK-AVX2-O0-NEXT: addq $56, %rsp
+; CHECK-AVX2-O0-NEXT: popq %rbx
+; CHECK-AVX2-O0-NEXT: popq %r12
+; CHECK-AVX2-O0-NEXT: popq %r13
+; CHECK-AVX2-O0-NEXT: popq %r14
+; CHECK-AVX2-O0-NEXT: popq %r15
+; CHECK-AVX2-O0-NEXT: popq %rbp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $120, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm8
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
+; CHECK-AVX512-O0-NEXT: pushq %rbp
+; CHECK-AVX512-O0-NEXT: pushq %r15
+; CHECK-AVX512-O0-NEXT: pushq %r14
+; CHECK-AVX512-O0-NEXT: pushq %r13
+; CHECK-AVX512-O0-NEXT: pushq %r12
+; CHECK-AVX512-O0-NEXT: pushq %rbx
+; CHECK-AVX512-O0-NEXT: subq $56, %rsp
+; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %cx
+; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %dx
+; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %di
+; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r8w
+; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r9w
+; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r10w
+; CHECK-AVX512-O0-NEXT: vmovd %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r11w
; CHECK-AVX512-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vzeroupper
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %bx
+; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %bp
+; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r14w
+; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r15w
+; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r12w
+; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r13w
+; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovd %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX512-O0-NEXT: movw %r13w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r12w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r15w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r14w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %bp, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r11w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r10w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r8w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %di, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %cx, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT: movl $32, %edi
; CHECK-AVX512-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
+; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX512-O0-NEXT: addq $120, %rsp
+; CHECK-AVX512-O0-NEXT: addq $56, %rsp
+; CHECK-AVX512-O0-NEXT: popq %rbx
+; CHECK-AVX512-O0-NEXT: popq %r12
+; CHECK-AVX512-O0-NEXT: popq %r13
+; CHECK-AVX512-O0-NEXT: popq %r14
+; CHECK-AVX512-O0-NEXT: popq %r15
+; CHECK-AVX512-O0-NEXT: popq %rbp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x bfloat> %v, ptr %x release, align 4
ret void
@@ -3632,593 +3064,235 @@ define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_align:
-; CHECK-SSE2-O0: # %bb.0:
-; CHECK-SSE2-O0-NEXT: subq $120, %rsp
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: xorps %xmm13, %xmm13
-; CHECK-SSE2-O0-NEXT: movaps %xmm13, %xmm15
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
-; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm14
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
-; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm11
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm11
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm7
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE2-O0-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
-; CHECK-SSE2-O0-NEXT: movl $32, %edi
-; CHECK-SSE2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
-; CHECK-SSE2-O0-NEXT: movl $3, %ecx
-; CHECK-SSE2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE2-O0-NEXT: addq $120, %rsp
-; CHECK-SSE2-O0-NEXT: retq
-;
-; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_align:
-; CHECK-SSE4-O0: # %bb.0:
-; CHECK-SSE4-O0-NEXT: subq $120, %rsp
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm15
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm14
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm12
-; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm11
-; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
-; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm13
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
-; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm7
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm2, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm6
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm6
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm3
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
-; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-SSE4-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-SSE4-O0-NEXT: movl $32, %edi
-; CHECK-SSE4-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
-; CHECK-SSE4-O0-NEXT: movl $3, %ecx
-; CHECK-SSE4-O0-NEXT: callq __atomic_store at PLT
-; CHECK-SSE4-O0-NEXT: addq $120, %rsp
-; CHECK-SSE4-O0-NEXT: retq
+; CHECK-SSE-O0-LABEL: store_atomic_vec16_bfloat_align:
+; CHECK-SSE-O0: # %bb.0:
+; CHECK-SSE-O0-NEXT: pushq %rbp
+; CHECK-SSE-O0-NEXT: pushq %r15
+; CHECK-SSE-O0-NEXT: pushq %r14
+; CHECK-SSE-O0-NEXT: pushq %r13
+; CHECK-SSE-O0-NEXT: pushq %r12
+; CHECK-SSE-O0-NEXT: pushq %rbx
+; CHECK-SSE-O0-NEXT: subq $56, %rsp
+; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
+; CHECK-SSE-O0-NEXT: pextrw $7, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-SSE-O0-NEXT: pextrw $6, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %cx
+; CHECK-SSE-O0-NEXT: pextrw $5, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %dx
+; CHECK-SSE-O0-NEXT: pextrw $4, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %di
+; CHECK-SSE-O0-NEXT: pextrw $3, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r8w
+; CHECK-SSE-O0-NEXT: pextrw $2, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r9w
+; CHECK-SSE-O0-NEXT: pextrw $1, %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r10w
+; CHECK-SSE-O0-NEXT: movd %xmm1, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r11w
+; CHECK-SSE-O0-NEXT: pextrw $7, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %bx
+; CHECK-SSE-O0-NEXT: pextrw $6, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %bp
+; CHECK-SSE-O0-NEXT: pextrw $5, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r14w
+; CHECK-SSE-O0-NEXT: pextrw $4, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r15w
+; CHECK-SSE-O0-NEXT: pextrw $3, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r12w
+; CHECK-SSE-O0-NEXT: pextrw $2, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: movw %ax, %r13w
+; CHECK-SSE-O0-NEXT: pextrw $1, %xmm0, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
+; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-SSE-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-SSE-O0-NEXT: movw %r13w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r12w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r15w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r14w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %bp, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r11w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r10w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %r8w, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %di, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %cx, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-SSE-O0-NEXT: movl $32, %edi
+; CHECK-SSE-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-SSE-O0-NEXT: movl $3, %ecx
+; CHECK-SSE-O0-NEXT: callq __atomic_store at PLT
+; CHECK-SSE-O0-NEXT: addq $56, %rsp
+; CHECK-SSE-O0-NEXT: popq %rbx
+; CHECK-SSE-O0-NEXT: popq %r12
+; CHECK-SSE-O0-NEXT: popq %r13
+; CHECK-SSE-O0-NEXT: popq %r14
+; CHECK-SSE-O0-NEXT: popq %r15
+; CHECK-SSE-O0-NEXT: popq %rbp
+; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX2-O0: # %bb.0:
-; CHECK-AVX2-O0-NEXT: subq $120, %rsp
-; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm8
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
+; CHECK-AVX2-O0-NEXT: pushq %rbp
+; CHECK-AVX2-O0-NEXT: pushq %r15
+; CHECK-AVX2-O0-NEXT: pushq %r14
+; CHECK-AVX2-O0-NEXT: pushq %r13
+; CHECK-AVX2-O0-NEXT: pushq %r12
+; CHECK-AVX2-O0-NEXT: pushq %rbx
+; CHECK-AVX2-O0-NEXT: subq $56, %rsp
+; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %cx
+; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %dx
+; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %di
+; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r8w
+; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r9w
+; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r10w
+; CHECK-AVX2-O0-NEXT: vmovd %xmm1, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r11w
; CHECK-AVX2-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX2-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vzeroupper
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %bx
+; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %bp
+; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r14w
+; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r15w
+; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r12w
+; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: movw %ax, %r13w
+; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovd %xmm0, %eax
+; CHECK-AVX2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX2-O0-NEXT: movw %r13w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r12w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r15w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r14w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %bp, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r11w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r10w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %r8w, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %di, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %cx, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $32, %edi
; CHECK-AVX2-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
+; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX2-O0-NEXT: addq $120, %rsp
+; CHECK-AVX2-O0-NEXT: addq $56, %rsp
+; CHECK-AVX2-O0-NEXT: popq %rbx
+; CHECK-AVX2-O0-NEXT: popq %r12
+; CHECK-AVX2-O0-NEXT: popq %r13
+; CHECK-AVX2-O0-NEXT: popq %r14
+; CHECK-AVX2-O0-NEXT: popq %r15
+; CHECK-AVX2-O0-NEXT: popq %rbp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX512-O0: # %bb.0:
-; CHECK-AVX512-O0-NEXT: subq $120, %rsp
-; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm8
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
-; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
-; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
+; CHECK-AVX512-O0-NEXT: pushq %rbp
+; CHECK-AVX512-O0-NEXT: pushq %r15
+; CHECK-AVX512-O0-NEXT: pushq %r14
+; CHECK-AVX512-O0-NEXT: pushq %r13
+; CHECK-AVX512-O0-NEXT: pushq %r12
+; CHECK-AVX512-O0-NEXT: pushq %rbx
+; CHECK-AVX512-O0-NEXT: subq $56, %rsp
+; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
+; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %cx
+; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %dx
+; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %di
+; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r8w
+; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r9w
+; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r10w
+; CHECK-AVX512-O0-NEXT: vmovd %xmm1, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r11w
; CHECK-AVX512-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
-; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
-; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm8, %xmm8
-; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
-; CHECK-AVX512-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vzeroupper
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %bx
+; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %bp
+; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r14w
+; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r15w
+; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r12w
+; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: movw %ax, %r13w
+; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovd %xmm0, %eax
+; CHECK-AVX512-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
+; CHECK-AVX512-O0-NEXT: movw %r13w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r12w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r15w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r14w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %bp, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r11w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r10w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r9w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %r8w, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %di, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %cx, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT: movl $32, %edi
; CHECK-AVX512-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
+; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store at PLT
-; CHECK-AVX512-O0-NEXT: addq $120, %rsp
+; CHECK-AVX512-O0-NEXT: addq $56, %rsp
+; CHECK-AVX512-O0-NEXT: popq %rbx
+; CHECK-AVX512-O0-NEXT: popq %r12
+; CHECK-AVX512-O0-NEXT: popq %r13
+; CHECK-AVX512-O0-NEXT: popq %r14
+; CHECK-AVX512-O0-NEXT: popq %r15
+; CHECK-AVX512-O0-NEXT: popq %rbp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x bfloat> %v, ptr %x release, align 32
ret void
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll
index ed882cbbc5254..096dc7fbf3fc2 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll
@@ -791,20 +791,9 @@ define double @load_double_seq_cst(ptr %fptr) {
define void @store_bfloat(ptr %fptr, bfloat %v) {
; X86-SSE1-LABEL: store_bfloat:
; X86-SSE1: # %bb.0:
-; X86-SSE1-NEXT: pushl %esi
-; X86-SSE1-NEXT: .cfi_def_cfa_offset 8
-; X86-SSE1-NEXT: subl $8, %esp
-; X86-SSE1-NEXT: .cfi_def_cfa_offset 16
-; X86-SSE1-NEXT: .cfi_offset %esi, -8
-; X86-SSE1-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE1-NEXT: movss %xmm0, (%esp)
-; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SSE1-NEXT: calll __truncsfbf2
-; X86-SSE1-NEXT: movw %ax, (%esi)
-; X86-SSE1-NEXT: addl $8, %esp
-; X86-SSE1-NEXT: .cfi_def_cfa_offset 8
-; X86-SSE1-NEXT: popl %esi
-; X86-SSE1-NEXT: .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT: movw %ax, (%ecx)
; X86-SSE1-NEXT: retl
;
; X86-SSE2-LABEL: store_bfloat:
@@ -823,20 +812,9 @@ define void @store_bfloat(ptr %fptr, bfloat %v) {
;
; X86-NOSSE-LABEL: store_bfloat:
; X86-NOSSE: # %bb.0:
-; X86-NOSSE-NEXT: pushl %esi
-; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8
-; X86-NOSSE-NEXT: subl $8, %esp
-; X86-NOSSE-NEXT: .cfi_def_cfa_offset 16
-; X86-NOSSE-NEXT: .cfi_offset %esi, -8
-; X86-NOSSE-NEXT: flds {{[0-9]+}}(%esp)
-; X86-NOSSE-NEXT: fstps (%esp)
-; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NOSSE-NEXT: calll __truncsfbf2
-; X86-NOSSE-NEXT: movw %ax, (%esi)
-; X86-NOSSE-NEXT: addl $8, %esp
-; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8
-; X86-NOSSE-NEXT: popl %esi
-; X86-NOSSE-NEXT: .cfi_def_cfa_offset 4
+; X86-NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT: movw %ax, (%ecx)
; X86-NOSSE-NEXT: retl
;
; X64-SSE-LABEL: store_bfloat:
@@ -857,15 +835,8 @@ define void @store_bfloat(ptr %fptr, bfloat %v) {
define bfloat @load_bfloat(ptr %fptr) {
; X86-SSE1-LABEL: load_bfloat:
; X86-SSE1: # %bb.0:
-; X86-SSE1-NEXT: pushl %eax
-; X86-SSE1-NEXT: .cfi_def_cfa_offset 8
; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE1-NEXT: movzwl (%eax), %eax
-; X86-SSE1-NEXT: shll $16, %eax
-; X86-SSE1-NEXT: movl %eax, (%esp)
-; X86-SSE1-NEXT: flds (%esp)
-; X86-SSE1-NEXT: popl %eax
-; X86-SSE1-NEXT: .cfi_def_cfa_offset 4
; X86-SSE1-NEXT: retl
;
; X86-SSE2-LABEL: load_bfloat:
@@ -884,15 +855,8 @@ define bfloat @load_bfloat(ptr %fptr) {
;
; X86-NOSSE-LABEL: load_bfloat:
; X86-NOSSE: # %bb.0:
-; X86-NOSSE-NEXT: pushl %eax
-; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8
; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOSSE-NEXT: movzwl (%eax), %eax
-; X86-NOSSE-NEXT: shll $16, %eax
-; X86-NOSSE-NEXT: movl %eax, (%esp)
-; X86-NOSSE-NEXT: flds (%esp)
-; X86-NOSSE-NEXT: popl %eax
-; X86-NOSSE-NEXT: .cfi_def_cfa_offset 4
; X86-NOSSE-NEXT: retl
;
; X64-SSE-LABEL: load_bfloat:
diff --git a/llvm/test/CodeGen/X86/ldexp.ll b/llvm/test/CodeGen/X86/ldexp.ll
index b6f2793c4b1f7..69e889ab422b9 100644
--- a/llvm/test/CodeGen/X86/ldexp.ll
+++ b/llvm/test/CodeGen/X86/ldexp.ll
@@ -65,7 +65,7 @@ define bfloat @ldexp_bf16(i8 zeroext %x) nounwind {
;
; WIN32-LABEL: ldexp_bf16:
; WIN32: # %bb.0:
-; WIN32-NEXT: subl $20, %esp
+; WIN32-NEXT: subl $16, %esp
; WIN32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp)
; WIN32-NEXT: fld1
@@ -75,11 +75,7 @@ define bfloat @ldexp_bf16(i8 zeroext %x) nounwind {
; WIN32-NEXT: flds {{[0-9]+}}(%esp)
; WIN32-NEXT: fstps (%esp)
; WIN32-NEXT: calll ___truncsfbf2
-; WIN32-NEXT: # kill: def $ax killed $ax def $eax
-; WIN32-NEXT: shll $16, %eax
-; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; WIN32-NEXT: flds {{[0-9]+}}(%esp)
-; WIN32-NEXT: addl $20, %esp
+; WIN32-NEXT: addl $16, %esp
; WIN32-NEXT: retl
%zext = zext i8 %x to i32
%ldexp = call bfloat @llvm.ldexp.bf16.i32(bfloat 1.000000e+00, i32 %zext)
diff --git a/llvm/test/CodeGen/X86/llvm.frexp.ll b/llvm/test/CodeGen/X86/llvm.frexp.ll
index fb778add339fb..b0a4dc638c755 100644
--- a/llvm/test/CodeGen/X86/llvm.frexp.ll
+++ b/llvm/test/CodeGen/X86/llvm.frexp.ll
@@ -133,13 +133,10 @@ define { bfloat, i32 } @test_frexp_bf16_i32(bfloat %a) nounwind {
; WIN32-LABEL: test_frexp_bf16_i32:
; WIN32: # %bb.0:
; WIN32-NEXT: pushl %esi
-; WIN32-NEXT: subl $28, %esp
-; WIN32-NEXT: flds {{[0-9]+}}(%esp)
-; WIN32-NEXT: fstps (%esp)
-; WIN32-NEXT: calll ___truncsfbf2
-; WIN32-NEXT: # kill: def $ax killed $ax def $eax
-; WIN32-NEXT: leal {{[0-9]+}}(%esp), %ecx
-; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; WIN32-NEXT: subl $24, %esp
+; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax
; WIN32-NEXT: shll $16, %eax
; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp)
; WIN32-NEXT: flds {{[0-9]+}}(%esp)
@@ -150,12 +147,8 @@ define { bfloat, i32 } @test_frexp_bf16_i32(bfloat %a) nounwind {
; WIN32-NEXT: fstps (%esp)
; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi
; WIN32-NEXT: calll ___truncsfbf2
-; WIN32-NEXT: # kill: def $ax killed $ax def $eax
-; WIN32-NEXT: shll $16, %eax
-; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; WIN32-NEXT: flds {{[0-9]+}}(%esp)
-; WIN32-NEXT: movl %esi, %eax
-; WIN32-NEXT: addl $28, %esp
+; WIN32-NEXT: movl %esi, %edx
+; WIN32-NEXT: addl $24, %esp
; WIN32-NEXT: popl %esi
; WIN32-NEXT: retl
%result = call { bfloat, i32 } @llvm.frexp.bf16.i32(bfloat %a)
diff --git a/llvm/test/CodeGen/X86/select-phi-s16-fp.ll b/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
index 083f418b6752e..c5dc4ddb091bc 100644
--- a/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
+++ b/llvm/test/CodeGen/X86/select-phi-s16-fp.ll
@@ -12,17 +12,10 @@
define void @phi_vec1bf16_to_f32_with_const_folding(ptr %dst) #0 {
; CHECK-LABEL: phi_vec1bf16_to_f32_with_const_folding:
; CHECK: ## %bb.0: ## %entry
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: jmp LBB0_1
; CHECK-NEXT: LBB0_1: ## %bb
-; CHECK-NEXT: xorps %xmm0, %xmm0
-; CHECK-NEXT: callq ___truncsfbf2
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: movw %ax, 2(%rbx)
-; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: movw %ax, 2(%rdi)
; CHECK-NEXT: retq
entry:
br label %bb
@@ -41,19 +34,10 @@ bb:
define void @phi_vec1bf16_to_f32(ptr %src, ptr %dst) #0 {
; CHECK-LABEL: phi_vec1bf16_to_f32:
; CHECK: ## %bb.0: ## %entry
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: movq %rsi, %rbx
-; CHECK-NEXT: movzwl (%rdi), %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: movw (%rdi), %ax
; CHECK-NEXT: jmp LBB1_1
; CHECK-NEXT: LBB1_1: ## %bb
-; CHECK-NEXT: callq ___truncsfbf2
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: movw %ax, 2(%rbx)
-; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: movw %ax, 2(%rsi)
; CHECK-NEXT: retq
entry:
%input = load <1 x bfloat>, ptr %src
>From 4ce805ab02a6b67e75c169e79d3c515af92310da Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Mon, 21 Sep 2026 13:58:12 +0200
Subject: [PATCH 4/5] add test that exposed libcall on riscv/loongarch at
higher opt levels
---
llvm/test/CodeGen/Generic/bfloat.ll | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/llvm/test/CodeGen/Generic/bfloat.ll b/llvm/test/CodeGen/Generic/bfloat.ll
index 5c41e2fc57178..35233e73c076e 100644
--- a/llvm/test/CodeGen/Generic/bfloat.ll
+++ b/llvm/test/CodeGen/Generic/bfloat.ll
@@ -67,6 +67,21 @@ define i16 @to_bits(bfloat %f) nounwind {
ret i16 %bits
}
+; Previously exposed a libcall at higher optimization levels.
+define i16 @to_bits_branchy(bfloat %x, i1 %c) {
+; ALL-LABEL: to_bits_branchy{{"?}}:
+; CHECK-NOT: __extend
+; CHECK-NOT: __trunc
+; CHECK-NOT: __gnu
+entry:
+ br i1 %c, label %a, label %b
+a:
+ %r = bitcast bfloat %x to i16
+ ret i16 %r
+b:
+ ret i16 0
+}
+
define bfloat @check_freeze(bfloat %f) nounwind {
; ALL-LABEL: check_freeze{{"?}}:
%t0 = freeze bfloat %f
>From 7d821f0781ce7299642d914adf24f33e76916d05 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Thu, 24 Sep 2026 21:04:09 +0200
Subject: [PATCH 5/5] use the riscv approach for bf16 on arm
---
llvm/include/llvm/CodeGen/TargetLowering.h | 6 --
llvm/lib/CodeGen/TargetLoweringBase.cpp | 12 +---
llvm/lib/Target/ARM/ARMISelLowering.cpp | 20 ++++++
llvm/lib/Target/ARM/ARMISelLowering.h | 8 ++-
llvm/test/CodeGen/ARM/atomic-load-store.ll | 9 +--
llvm/test/CodeGen/ARM/bf16-instructions.ll | 62 ++++++++++++-------
llvm/test/CodeGen/Thumb2/bf16-instructions.ll | 19 +++---
7 files changed, 77 insertions(+), 59 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 1551f833cbccf..c08d0e53ec34b 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -571,12 +571,6 @@ class LLVM_ABI TargetLoweringBase {
// values are always passed and returned as f32.
virtual bool useFPRegsForHalfType() const { return false; }
- // Return true if, for soft-promoted bfloat, the bfloat type should be
- // passed to and returned from functions as f32. The default behavior is to
- // pass as i16. If soft-promoted bfloat is not used, this function is
- // ignored and values are always passed and returned as f32.
- virtual bool useFPRegsForBFloat16Type() const { return false; }
-
// There are two general methods for expanding a BUILD_VECTOR node:
// 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
// them together.
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index d9f24f1f70a83..3808497f512ff 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1536,16 +1536,8 @@ void TargetLoweringBase::computeRegisterProperties(
// promote it to f32, because there are no bf16 library calls (except for
// converting from f32 to bf16).
if (!isTypeLegal(MVT::bf16)) {
- // Allow targets to control how we legalize bfloat.
- bool UseFPRegsForBFloat16Type = useFPRegsForBFloat16Type();
-
- if (!UseFPRegsForBFloat16Type) {
- NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::i16];
- RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::i16];
- } else {
- NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
- RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
- }
+ NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::i16];
+ RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::i16];
TransformToType[MVT::bf16] = MVT::f32;
ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
}
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 194b597bc77ba..5e2bf7ce1b7a6 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -22260,6 +22260,26 @@ static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
return (Members > 0 && Members <= 4);
}
+MVT ARMTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC,
+ EVT VT) const {
+ // Pass soft-promoted bf16 like f32.
+ if (VT == MVT::bf16 && !isTypeLegal(MVT::bf16))
+ return TargetLowering::getRegisterTypeForCallingConv(Context, CC, MVT::f32);
+
+ return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
+}
+
+unsigned ARMTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC,
+ EVT VT) const {
+ // Pass soft-promoted bf16 like f32.
+ if (VT == MVT::bf16 && !isTypeLegal(MVT::bf16))
+ return TargetLowering::getNumRegistersForCallingConv(Context, CC, MVT::f32);
+
+ return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
+}
+
/// Return the correct alignment for the current calling convention.
Align ARMTargetLowering::getABIAlignmentForCallingConv(
Type *ArgTy, const DataLayout &DL) const {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 30838f1772b4a..fa0efbebe9cf1 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -504,7 +504,13 @@ class VectorType;
Value *Accumulator = nullptr) const override;
bool useFPRegsForHalfType() const override { return true; }
- bool useFPRegsForBFloat16Type() const override { return true; }
+
+ MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
+ EVT VT) const override;
+
+ unsigned getNumRegistersForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC,
+ EVT VT) const override;
protected:
std::pair<const TargetRegisterClass *, uint8_t>
diff --git a/llvm/test/CodeGen/ARM/atomic-load-store.ll b/llvm/test/CodeGen/ARM/atomic-load-store.ll
index 0c787a4ca05c3..28cc6b721d46c 100644
--- a/llvm/test/CodeGen/ARM/atomic-load-store.ll
+++ b/llvm/test/CodeGen/ARM/atomic-load-store.ll
@@ -733,16 +733,9 @@ define void @store_atomic_bf16__seq_cst(ptr %ptr, bfloat %val1) {
;
; ARMOPTNONE-LABEL: store_atomic_bf16__seq_cst:
; ARMOPTNONE: @ %bb.0:
-; ARMOPTNONE-NEXT: sub sp, sp, #4
-; ARMOPTNONE-NEXT: str r1, [sp] @ 4-byte Spill
-; ARMOPTNONE-NEXT: mov r1, r0
-; ARMOPTNONE-NEXT: ldr r0, [sp] @ 4-byte Reload
-; ARMOPTNONE-NEXT: vmov s0, r0
-; ARMOPTNONE-NEXT: vmov r0, s0
; ARMOPTNONE-NEXT: dmb ish
-; ARMOPTNONE-NEXT: strh r0, [r1]
+; ARMOPTNONE-NEXT: strh r1, [r0]
; ARMOPTNONE-NEXT: dmb ish
-; ARMOPTNONE-NEXT: add sp, sp, #4
; ARMOPTNONE-NEXT: bx lr
;
; THUMBTWO-LABEL: store_atomic_bf16__seq_cst:
diff --git a/llvm/test/CodeGen/ARM/bf16-instructions.ll b/llvm/test/CodeGen/ARM/bf16-instructions.ll
index e6527050d5228..5f2337219ae2f 100644
--- a/llvm/test/CodeGen/ARM/bf16-instructions.ll
+++ b/llvm/test/CodeGen/ARM/bf16-instructions.ll
@@ -260,28 +260,46 @@ define bfloat @test_tailcall_flipped(bfloat %a, bfloat %b) #0 {
;}
define bfloat @test_phi(ptr %p1) #0 {
-; CHECK-LABEL: test_phi:
-; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: ldrh r0, [r0]
-; CHECK-NEXT: vmov s18, r0
-; CHECK-NEXT: .LBB5_1: @ %loop
-; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldrh r0, [r4]
-; CHECK-NEXT: vmov.f32 s16, s18
-; CHECK-NEXT: vmov s18, r0
-; CHECK-NEXT: mov r0, r4
-; CHECK-NEXT: bl test_dummy
-; CHECK-NEXT: tst r0, #1
-; CHECK-NEXT: bne .LBB5_1
-; CHECK-NEXT: @ %bb.2: @ %return
-; CHECK-NEXT: vmov.f32 s0, s16
-; CHECK-NEXT: vpop {d8, d9}
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-CVT-LABEL: test_phi:
+; CHECK-CVT: @ %bb.0: @ %entry
+; CHECK-CVT-NEXT: .save {r4, r5, r6, lr}
+; CHECK-CVT-NEXT: push {r4, r5, r6, lr}
+; CHECK-CVT-NEXT: ldrh r5, [r0]
+; CHECK-CVT-NEXT: mov r4, r0
+; CHECK-CVT-NEXT: .LBB5_1: @ %loop
+; CHECK-CVT-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-CVT-NEXT: mov r0, r4
+; CHECK-CVT-NEXT: mov r6, r5
+; CHECK-CVT-NEXT: ldrh r5, [r4]
+; CHECK-CVT-NEXT: bl test_dummy
+; CHECK-CVT-NEXT: tst r0, #1
+; CHECK-CVT-NEXT: bne .LBB5_1
+; CHECK-CVT-NEXT: @ %bb.2: @ %return
+; CHECK-CVT-NEXT: vmov s0, r6
+; CHECK-CVT-NEXT: pop {r4, r5, r6, pc}
+;
+; CHECK-BF16-LABEL: test_phi:
+; CHECK-BF16: @ %bb.0: @ %entry
+; CHECK-BF16-NEXT: .save {r4, lr}
+; CHECK-BF16-NEXT: push {r4, lr}
+; CHECK-BF16-NEXT: .vsave {d8, d9}
+; CHECK-BF16-NEXT: vpush {d8, d9}
+; CHECK-BF16-NEXT: mov r4, r0
+; CHECK-BF16-NEXT: ldrh r0, [r0]
+; CHECK-BF16-NEXT: vmov s18, r0
+; CHECK-BF16-NEXT: .LBB5_1: @ %loop
+; CHECK-BF16-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-BF16-NEXT: ldrh r0, [r4]
+; CHECK-BF16-NEXT: vmov.f32 s16, s18
+; CHECK-BF16-NEXT: vmov s18, r0
+; CHECK-BF16-NEXT: mov r0, r4
+; CHECK-BF16-NEXT: bl test_dummy
+; CHECK-BF16-NEXT: tst r0, #1
+; CHECK-BF16-NEXT: bne .LBB5_1
+; CHECK-BF16-NEXT: @ %bb.2: @ %return
+; CHECK-BF16-NEXT: vmov.f32 s0, s16
+; CHECK-BF16-NEXT: vpop {d8, d9}
+; CHECK-BF16-NEXT: pop {r4, pc}
entry:
%a = load bfloat, ptr %p1
br label %loop
diff --git a/llvm/test/CodeGen/Thumb2/bf16-instructions.ll b/llvm/test/CodeGen/Thumb2/bf16-instructions.ll
index de4c1bce22bcd..f3852c6a2cd9c 100644
--- a/llvm/test/CodeGen/Thumb2/bf16-instructions.ll
+++ b/llvm/test/CodeGen/Thumb2/bf16-instructions.ll
@@ -1045,26 +1045,21 @@ define bfloat @test_phi(ptr %p1) {
;
; CHECK-FP-LABEL: test_phi:
; CHECK-FP: @ %bb.0: @ %entry
-; CHECK-FP-NEXT: .save {r4, lr}
-; CHECK-FP-NEXT: push {r4, lr}
-; CHECK-FP-NEXT: .vsave {d8, d9}
-; CHECK-FP-NEXT: vpush {d8, d9}
+; CHECK-FP-NEXT: .save {r4, r5, r6, lr}
+; CHECK-FP-NEXT: push {r4, r5, r6, lr}
+; CHECK-FP-NEXT: ldrh r5, [r0]
; CHECK-FP-NEXT: mov r4, r0
-; CHECK-FP-NEXT: ldrh r0, [r0]
-; CHECK-FP-NEXT: vmov s18, r0
; CHECK-FP-NEXT: .LBB36_1: @ %loop
; CHECK-FP-NEXT: @ =>This Inner Loop Header: Depth=1
-; CHECK-FP-NEXT: ldrh r0, [r4]
-; CHECK-FP-NEXT: vmov.f32 s16, s18
-; CHECK-FP-NEXT: vmov s18, r0
; CHECK-FP-NEXT: mov r0, r4
+; CHECK-FP-NEXT: mov r6, r5
+; CHECK-FP-NEXT: ldrh r5, [r4]
; CHECK-FP-NEXT: bl test_dummy
; CHECK-FP-NEXT: lsls r0, r0, #31
; CHECK-FP-NEXT: bne .LBB36_1
; CHECK-FP-NEXT: @ %bb.2: @ %return
-; CHECK-FP-NEXT: vmov.f32 s0, s16
-; CHECK-FP-NEXT: vpop {d8, d9}
-; CHECK-FP-NEXT: pop {r4, pc}
+; CHECK-FP-NEXT: vmov s0, r6
+; CHECK-FP-NEXT: pop {r4, r5, r6, pc}
entry:
%a = load bfloat, ptr %p1
br label %loop
More information about the llvm-commits
mailing list